Compare commits
140
Commits
8962454530
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3a9deb8cdb | ||
|
|
4653b17e18 | ||
|
|
bdb927f83d | ||
|
|
49ee611847 | ||
|
|
72ea2988be | ||
|
|
9294defef8 | ||
|
|
f024be6e04 | ||
|
|
b053dc66bf | ||
|
|
c0b1225f6d | ||
|
|
5ba88c69fa | ||
|
|
8635560993 | ||
|
|
f9be71b9ad | ||
|
|
307dd23d6d | ||
|
|
dcaf843b8a | ||
|
|
7ed7da1398 | ||
|
|
1316a405a6 | ||
|
|
a0754ab06d | ||
|
|
adaf525a88 | ||
|
|
a8d86bb10b | ||
|
|
9deefc6370 | ||
|
|
d7098c4b27 | ||
|
|
d21949c041 | ||
|
|
7806a42c45 | ||
|
|
f71250c3dd | ||
|
|
ba2806b985 | ||
|
|
9497dda929 | ||
|
|
3b4ba29bec | ||
|
|
3eb48c949f | ||
|
|
5f13bf0aff | ||
|
|
73800225c3 | ||
|
|
becff66e0a | ||
|
|
2aebf8f40e | ||
|
|
0ca7a83e0b | ||
|
|
d16c3db9d2 | ||
|
|
39ec294853 | ||
|
|
6ba65a12f2 | ||
|
|
ea24e7890f | ||
|
|
0714f7a509 | ||
|
|
d03f87b00d | ||
|
|
51fa3a73ab | ||
|
|
383176a7ea | ||
|
|
671d56a08a | ||
|
|
b7f46559f0 | ||
|
|
abc022eece | ||
|
|
87eca0c9a9 | ||
|
|
e5ccf48468 | ||
|
|
c7c7268db0 | ||
|
|
d3ef6fa44c | ||
|
|
1af6efb356 | ||
|
|
782aa9f737 | ||
|
|
a96226f228 | ||
|
|
5c1c6f56c9 | ||
|
|
3c655d83e1 | ||
|
|
e63459dd70 | ||
|
|
24b833d19d | ||
|
|
79108a72a0 | ||
|
|
930b13ff47 | ||
|
|
39531a9972 | ||
|
|
a97ba661d9 | ||
|
|
bd33abf3dd | ||
|
|
1f008aeb18 | ||
|
|
26a0f45269 | ||
|
|
9bd51f3c6c | ||
|
|
8c78bcf9ce | ||
|
|
858b15a1a3 | ||
|
|
b3d6d65db1 | ||
|
|
c48d68ac92 | ||
|
|
af0a32a780 | ||
|
|
95736b0eab | ||
|
|
037bc32a6f | ||
|
|
22043a3c86 | ||
|
|
719ca7ae0b | ||
|
|
bdb6efa8c6 | ||
|
|
a10aeb9843 | ||
|
|
5ebb3520ec | ||
|
|
70bc7ad3db | ||
|
|
086a5eb7a0 | ||
|
|
01a541cadf | ||
|
|
569a9e7ae8 | ||
|
|
a076ccfe51 | ||
|
|
8d24e7bbb5 | ||
|
|
3a486014fc | ||
|
|
d9173d49b8 | ||
|
|
fb5bb1e3de | ||
|
|
aaca73afb2 | ||
|
|
06fa70d081 | ||
|
|
5981f0ae85 | ||
|
|
c9582ce041 | ||
|
|
e99c2b22cb | ||
|
|
617f95f2bf | ||
|
|
06e86ccd05 | ||
|
|
386e306642 | ||
|
|
f91164863b | ||
|
|
54db7c3458 | ||
|
|
96b05fbd85 | ||
|
|
3a3317b487 | ||
|
|
24bc736393 | ||
|
|
33e63ee5b4 | ||
|
|
5c128c1c18 | ||
|
|
c02a8ec4c8 | ||
|
|
983fd839eb | ||
|
|
1455d9560e | ||
|
|
65ae6a9f89 | ||
|
|
596f0b4532 | ||
|
|
d452a47be0 | ||
|
|
99923bed12 | ||
|
|
434bac93ce | ||
|
|
1e371cd829 | ||
|
|
157bb18fa8 | ||
|
|
6fd0683425 | ||
|
|
193e669bf1 | ||
|
|
20bdf96b31 | ||
|
|
8368516253 | ||
|
|
c76c64a4b5 | ||
|
|
179f73c18f | ||
|
|
505db85539 | ||
|
|
09d347a045 | ||
|
|
e80a4b1642 | ||
|
|
35babebbd1 | ||
|
|
4b7ad9a387 | ||
|
|
8fdbf9ca90 | ||
|
|
db0069f262 | ||
|
|
960539066e | ||
|
|
d2c6e8fc5a | ||
|
|
d0b0c7b02a | ||
|
|
b75a17e51b | ||
|
|
c4f57432e1 | ||
|
|
a9c3bd8e6d | ||
|
|
74c59661b1 | ||
|
|
a7903562b4 | ||
|
|
14246bcf01 | ||
|
|
2b489783ec | ||
|
|
3d871c992d | ||
|
|
a1a03f9880 | ||
|
|
eecbfd6693 | ||
|
|
d93f1e4eab | ||
|
|
69e1215424 | ||
|
|
5e3ae7b835 | ||
|
|
bc3709a66a | ||
|
|
6b44d7c414 |
@@ -45,6 +45,10 @@ mesh-lab diagram --live <instance> draw what is actually standing
|
||||
correctly and snapshots roughly 76× slower — which does not make the lab slow, it makes it
|
||||
unused, and a warning about that is read once and ignored forever.
|
||||
|
||||
Behind a VPN client that routes every private range, incus finds no free range for the uplink a
|
||||
machine fetches its images over, and nothing can be raised. Name one the host does not route:
|
||||
`MESH_LAB_UPLINK_V4=192.168.231.1/24`.
|
||||
|
||||
If the incus socket is not reachable as your user — the group was granted to a session that
|
||||
already existed — set `MESH_LAB_INCUS="sudo -n incus"`.
|
||||
|
||||
@@ -144,10 +148,10 @@ is written down here rather than reconstructed a third time.
|
||||
# built for \"\", which is not a system it knows" — which reads like a broken bundle and is not
|
||||
# (novox/hq 04-ISSUES/146).
|
||||
export MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host
|
||||
# The bundle that raises the bus the mesh runs on. The other one in that directory raises the
|
||||
# predecessor's broker and a control plane that crash-loops on a missing MESH_BUS_NATS — which
|
||||
# looks like a broken lab and is a bundle nobody moved (novox/hq 04-ISSUES/146).
|
||||
export MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node-nats.lock
|
||||
# The one genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331). The bundle that raised the
|
||||
# predecessor's broker, and a control plane that crash-looped on a missing MESH_BUS_NATS, is gone
|
||||
# (novox/hq issue 146).
|
||||
export MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||
export MESH_LAB_MODULES=<mesh-controller>/examples/modules
|
||||
export MESH_LAB_BUILDER=<mesh-controller>/build/mesh-builder # build/, which is git-ignored
|
||||
|
||||
@@ -194,8 +198,12 @@ runs, so the receipt names the catalogue's commit too, and a run taken before a
|
||||
says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
|
||||
scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
|
||||
source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
|
||||
beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
|
||||
where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
|
||||
beside this one), and rebuilt by `scripts/build-module-runtime.sh` where the image is older, missing,
|
||||
or the source is uncommitted (novox/hq 04-ISSUES/075). The image is today's tool runtime: `node-tools`,
|
||||
built with Go from mesh-tools' `node-tools/`, launching the module's compiled bundle through the
|
||||
launcher the mesh's builder writes beside each entrypoint (novox/hq ADR 0193) — so building one needs
|
||||
`go`, and `npm` for the SDK's compiler. It serves on the node's runtime credential or a
|
||||
`MESH_BROKER_URL`, never the served module's own: the runtime refuses that.
|
||||
`--no-build` skips this too, and then the bed runs whatever image the store holds.
|
||||
|
||||
Check before running a long suite — it says which of these are missing rather than skipping
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// labAsker is the module `lab-asker` asking the operator, as any module does: the SDK's asker client
|
||||
// (mesh-sdk go/asker) over the lab's bus, on that module's own composed credential. What it performs on a
|
||||
// warrant is one switch flipped, bound when it asked, and counted: the proof that an approval does exactly
|
||||
// that one thing, once.
|
||||
type labAsker struct {
|
||||
t *testing.T
|
||||
client *asker.Client
|
||||
nc *nats.Conn
|
||||
|
||||
mu sync.Mutex
|
||||
warrants []asks.Warrant
|
||||
handled []asker.Handled
|
||||
acts []map[string]string // what was performed
|
||||
}
|
||||
|
||||
// flip is the act an approve option binds.
|
||||
func flip(which string) asks.Act {
|
||||
return asks.Act{"verb": "lab.flip", "arg.switch": which}
|
||||
}
|
||||
|
||||
func newLabAsker(t *testing.T, l *lab) *labAsker {
|
||||
nc, _ := l.as(machine + ".lab-asker")
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &labAsker{t: t, nc: nc}
|
||||
a.client = &asker.Client{Module: "lab-asker", State: &memState{vals: map[string]json.RawMessage{}, revs: map[string]uint64{}},
|
||||
Now: time.Now,
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(subject, raw, nats.MsgId("lab-asker."+id))
|
||||
return err
|
||||
}}
|
||||
// Its warrants, on the one subject the bus lets it hear them on.
|
||||
sub, err := nc.Subscribe(asks.DecidedSubject("lab-asker"), func(m *nats.Msg) {
|
||||
var w asks.Warrant
|
||||
if err := json.Unmarshal(m.Data, &w); err != nil {
|
||||
t.Errorf("a word on the asker's decided subject is not a warrant: %v", err)
|
||||
return
|
||||
}
|
||||
a.take(w)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
return a
|
||||
}
|
||||
|
||||
// take hands one warrant to the SDK's Take, which performs the option once, and only the act it bound.
|
||||
func (a *labAsker) take(w asks.Warrant) asker.Handled {
|
||||
h, err := a.client.Take(w, func(o asks.Option, w asks.Warrant) error {
|
||||
if o.Level == asks.Acknowledge {
|
||||
return nil // decline: nothing is performed
|
||||
}
|
||||
act := flip(o.ID)
|
||||
if err := o.Performs(act); err != nil {
|
||||
return err
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.acts = append(a.acts, act)
|
||||
a.mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
a.mu.Lock()
|
||||
a.warrants = append(a.warrants, w)
|
||||
a.handled = append(a.handled, h)
|
||||
a.mu.Unlock()
|
||||
if err != nil {
|
||||
a.t.Logf("the asker refused a warrant for %s: %v", w.Ask, err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// ask asks the operator: one approve option binding a switch, one decline that acknowledges.
|
||||
func (a *labAsker) ask(id, headline string, expires time.Duration) asks.Ask {
|
||||
a.t.Helper()
|
||||
binds, err := asks.ActDigest(flip("approve"))
|
||||
if err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
q := asks.Ask{ID: id, Headline: headline,
|
||||
Explanation: "Needs you: approve or decline. The lab asks whether to flip one switch.",
|
||||
Who: asks.Operator, Expires: time.Now().Add(expires), OnExpiry: "nothing is flipped",
|
||||
Options: []asks.Option{
|
||||
{ID: "approve", Label: "Approve", Does: "the switch is flipped, once", Level: asks.Approve, Binds: binds},
|
||||
{ID: "decline", Label: "Decline", Does: "nothing is done", Level: asks.Acknowledge},
|
||||
}}
|
||||
if err := a.client.Ask(q); err != nil {
|
||||
a.t.Fatalf("the asker could not ask %s: %v", id, err)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// warrantFor waits for the router's word on one ask.
|
||||
func (a *labAsker) warrantFor(id string, within time.Duration) asks.Warrant {
|
||||
a.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
a.mu.Lock()
|
||||
for _, w := range a.warrants {
|
||||
if w.Ask == id {
|
||||
a.mu.Unlock()
|
||||
return w
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
a.t.Fatalf("no word on the ask %s within %s", id, within)
|
||||
return asks.Warrant{}
|
||||
}
|
||||
|
||||
// noWordOn says the router said nothing on an ask for a while.
|
||||
func (a *labAsker) noWordOn(id string, wait time.Duration) bool {
|
||||
time.Sleep(wait)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for _, w := range a.warrants {
|
||||
if w.Ask == id {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *labAsker) performed() []map[string]string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return append([]map[string]string(nil), a.acts...)
|
||||
}
|
||||
|
||||
func (a *labAsker) wordsOn(id string) int {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
n := 0
|
||||
for _, w := range a.warrants {
|
||||
if w.Ask == id {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// memState is the asker's own record, kept in memory: the lab's asker restarts never.
|
||||
type memState struct {
|
||||
mu sync.Mutex
|
||||
vals map[string]json.RawMessage
|
||||
revs map[string]uint64
|
||||
n uint64
|
||||
}
|
||||
|
||||
func (m *memState) Get(k string) (*stdio.StateEntry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
v, ok := m.vals[k]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.StateEntry{Key: k, Value: v, Revision: m.revs[k]}, nil
|
||||
}
|
||||
|
||||
func (m *memState) Create(k string, v any) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, ok := m.vals[k]; ok {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return m.put(k, v)
|
||||
}
|
||||
|
||||
func (m *memState) Update(k string, v any, rev uint64) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if rev == 0 || m.revs[k] != rev {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return m.put(k, v)
|
||||
}
|
||||
|
||||
func (m *memState) put(k string, v any) (uint64, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.n++
|
||||
m.vals[k], m.revs[k] = raw, m.n
|
||||
return m.n, nil
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
// Package asks is the lab's proof of the operator's answers (novox/hq ADR 0259): the whole flow, from an ask
|
||||
// to the act its approval authorises, on a real bus composed by the controller, with a fake Telegram. The
|
||||
// proof is TestTheOperatorsAnswerEndToEnd; the live acceptance after rollout is live-acceptance.sh.
|
||||
package asks
|
||||
@@ -0,0 +1,351 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeTelegram is the Bot API as the channel uses it, and nothing more: the lab's phone. It holds the updates
|
||||
// the lab makes happen (a message typed, a button tapped) for the channel's long poll, and keeps every message
|
||||
// the bot sends or edits, by chat, so the lab reads what the operator would see. It never reaches Telegram.
|
||||
type fakeTelegram struct {
|
||||
t *testing.T
|
||||
server *httptest.Server
|
||||
token string
|
||||
|
||||
mu sync.Mutex
|
||||
updates []map[string]any
|
||||
next int64
|
||||
arrived chan struct{}
|
||||
sent []tgMessage
|
||||
msgID int64
|
||||
taps int // answerCallbackQuery calls
|
||||
deleted int
|
||||
// toasts are the words a tap's acknowledgement showed over the chat (answerCallbackQuery), by the chat the
|
||||
// tap came from: what the phone shows the tapper and never puts in the chat. Kept apart from sent, so a
|
||||
// chat's messages stay what the chat holds.
|
||||
toasts []tgMessage
|
||||
tappedIn map[string]int64 // a tap's callback id to the chat it came from
|
||||
}
|
||||
|
||||
// tgMessage is one message the bot sent or edited: its chat, its id, its words and its buttons (label →
|
||||
// callback data).
|
||||
type tgMessage struct {
|
||||
Chat int64
|
||||
ID int64
|
||||
Text string
|
||||
Buttons [][2]string
|
||||
Edited bool
|
||||
At time.Time
|
||||
}
|
||||
|
||||
func newFakeTelegram(t *testing.T, token string) *fakeTelegram {
|
||||
f := &fakeTelegram{t: t, token: token, next: 1, arrived: make(chan struct{}, 1)}
|
||||
f.server = httptest.NewServer(http.HandlerFunc(f.serve))
|
||||
t.Cleanup(f.server.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) URL() string { return f.server.URL }
|
||||
|
||||
func (f *fakeTelegram) serve(w http.ResponseWriter, r *http.Request) {
|
||||
prefix := "/bot" + f.token + "/"
|
||||
if !strings.HasPrefix(r.URL.Path, prefix) {
|
||||
// A wrong token is what the real API answers 401 to.
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": false, "error_code": 401, "description": "Unauthorized"})
|
||||
return
|
||||
}
|
||||
method := strings.TrimPrefix(r.URL.Path, prefix)
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
ok := func(result any) { _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "result": result}) }
|
||||
switch method {
|
||||
case "getMe":
|
||||
ok(map[string]any{"id": 1000, "is_bot": true, "first_name": "Lab", "username": "lab_mesh_bot"})
|
||||
case "getUpdates":
|
||||
offset := int64(num(body["offset"]))
|
||||
wait := time.Duration(num(body["timeout"])) * time.Second
|
||||
if wait > 2*time.Second {
|
||||
wait = 2 * time.Second // the lab's phone answers quickly; the channel polls again at once
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
f.mu.Lock()
|
||||
var out []map[string]any
|
||||
for _, u := range f.updates {
|
||||
if int64(num(u["update_id"])) >= offset {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if len(out) > 0 {
|
||||
f.t.Logf("phone: getUpdates from %d hands %d update(s)", offset, len(out))
|
||||
}
|
||||
if len(out) > 0 || time.Now().After(deadline) {
|
||||
ok(orEmpty(out))
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-f.arrived:
|
||||
case <-time.After(time.Until(deadline)):
|
||||
}
|
||||
}
|
||||
case "sendMessage", "editMessageText":
|
||||
chat := int64(num(body["chat_id"]))
|
||||
text, _ := body["text"].(string)
|
||||
if mode, _ := body["parse_mode"].(string); mode == "HTML" {
|
||||
// What a message shows, and what a tap on it carries back, is its text with the markup read: tags gone,
|
||||
// entities unescaped — as the service gives it (novox/hq issue 692).
|
||||
text = shownOfHTML(text)
|
||||
}
|
||||
m := tgMessage{Chat: chat, Text: text, Buttons: buttonsOf(body["reply_markup"]), At: time.Now()}
|
||||
f.mu.Lock()
|
||||
if method == "sendMessage" {
|
||||
f.msgID++
|
||||
m.ID = f.msgID
|
||||
} else {
|
||||
m.ID, m.Edited = int64(num(body["message_id"])), true
|
||||
}
|
||||
f.sent = append(f.sent, m)
|
||||
f.mu.Unlock()
|
||||
f.t.Logf("phone: %s to %d (#%d): %q %v", method, chat, m.ID, firstLine(text), m.Buttons)
|
||||
ok(map[string]any{"message_id": m.ID, "chat": map[string]any{"id": chat, "type": "private"}, "text": text})
|
||||
case "answerCallbackQuery":
|
||||
id, _ := body["callback_query_id"].(string)
|
||||
text, _ := body["text"].(string)
|
||||
f.mu.Lock()
|
||||
f.taps++
|
||||
chat, known := f.tappedIn[id]
|
||||
if known && text != "" {
|
||||
f.toasts = append(f.toasts, tgMessage{Chat: chat, Text: text, At: time.Now()})
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if text != "" {
|
||||
f.t.Logf("phone: answerCallbackQuery %s in %d: %q", id, chat, firstLine(text))
|
||||
}
|
||||
ok(true)
|
||||
case "deleteMessage":
|
||||
f.mu.Lock()
|
||||
f.deleted++
|
||||
f.mu.Unlock()
|
||||
ok(true)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": false, "error_code": 404, "description": "Not Found: " + method})
|
||||
}
|
||||
}
|
||||
|
||||
// push makes one thing happen on the phone, for the channel's next poll.
|
||||
func (f *fakeTelegram) push(u map[string]any) {
|
||||
f.mu.Lock()
|
||||
u["update_id"] = f.next
|
||||
f.next++
|
||||
f.updates = append(f.updates, u)
|
||||
f.mu.Unlock()
|
||||
select {
|
||||
case f.arrived <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// typed is a message an account writes in a chat with the bot; private when the chat is the account's own.
|
||||
func (f *fakeTelegram) typed(from int64, chat int64, chatType, text string) {
|
||||
f.push(map[string]any{"message": map[string]any{"message_id": 9000 + f.nextID(), "text": text,
|
||||
"from": map[string]any{"id": from, "is_bot": false, "first_name": fmt.Sprintf("Account %d", from)},
|
||||
"chat": map[string]any{"id": chat, "type": chatType}}})
|
||||
}
|
||||
|
||||
// tapped is a button tapped on a message the bot sent, carrying the button's data and the message's words as
|
||||
// the service gives them: the words the message shows now (its last send or edit), as on the phone.
|
||||
func (f *fakeTelegram) tapped(from int64, chat int64, chatType string, onMessage int64, data string) {
|
||||
f.tappedOn(from, chat, chatType, onMessage, data, f.textOf(onMessage))
|
||||
}
|
||||
|
||||
// tappedOn is a tap on a message showing the words given: what a message changed under the operator's thumb,
|
||||
// or forwarded and edited elsewhere, carries.
|
||||
func (f *fakeTelegram) tappedOn(from int64, chat int64, chatType string, onMessage int64, data, text string) {
|
||||
id := fmt.Sprintf("cb-%d", f.nextID())
|
||||
f.mu.Lock()
|
||||
if f.tappedIn == nil {
|
||||
f.tappedIn = map[string]int64{}
|
||||
}
|
||||
f.tappedIn[id] = chat
|
||||
f.mu.Unlock()
|
||||
f.push(map[string]any{"callback_query": map[string]any{"id": id, "data": data,
|
||||
"from": map[string]any{"id": from, "is_bot": false, "first_name": fmt.Sprintf("Account %d", from)},
|
||||
"message": map[string]any{"message_id": onMessage, "text": text,
|
||||
"chat": map[string]any{"id": chat, "type": chatType}}}})
|
||||
}
|
||||
|
||||
// textOf is what a message the bot sent shows now.
|
||||
func (f *fakeTelegram) textOf(id int64) string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
text := ""
|
||||
for _, m := range f.sent {
|
||||
if m.ID == id {
|
||||
text = m.Text
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) nextID() int64 {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.msgID++
|
||||
return f.msgID
|
||||
}
|
||||
|
||||
// waitFor waits for a message to a chat that satisfies match, sent after since.
|
||||
func (f *fakeTelegram) waitFor(what string, chat int64, since time.Time, within time.Duration, match func(tgMessage) bool) tgMessage {
|
||||
f.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
f.mu.Lock()
|
||||
for _, m := range f.sent {
|
||||
if m.Chat == chat && !m.At.Before(since) && match(m) {
|
||||
f.mu.Unlock()
|
||||
return m
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
f.t.Fatalf("the phone never showed %s (chat %d) within %s; it showed:\n%s", what, chat, within, f.transcript())
|
||||
return tgMessage{}
|
||||
}
|
||||
|
||||
// toldTapper waits for words the phone showed in a chat at or after a time, as a message in it or as a tap's
|
||||
// acknowledgement over it: where the channel says the router's word on a tap (issue 372).
|
||||
func (f *fakeTelegram) toldTapper(what string, chat int64, since time.Time, within time.Duration, match func(string) bool) string {
|
||||
f.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
f.mu.Lock()
|
||||
for _, m := range append(append([]tgMessage{}, f.sent...), f.toasts...) {
|
||||
if m.Chat == chat && !m.At.Before(since) && match(m.Text) {
|
||||
f.mu.Unlock()
|
||||
return m.Text
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
f.t.Fatalf("the phone never showed %s (chat %d) within %s; it showed:\n%s", what, chat, within, f.transcript())
|
||||
return ""
|
||||
}
|
||||
|
||||
// since is every message to a chat sent at or after a time.
|
||||
func (f *fakeTelegram) since(chat int64, at time.Time) []tgMessage {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []tgMessage
|
||||
for _, m := range f.sent {
|
||||
if m.Chat == chat && !m.At.Before(at) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeTelegram) transcript() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var b strings.Builder
|
||||
for _, m := range f.sent {
|
||||
fmt.Fprintf(&b, " %s to %d #%d edited=%v %q %v\n", m.At.Format("15:04:05.000"), m.Chat, m.ID, m.Edited,
|
||||
firstLine(m.Text), m.Buttons)
|
||||
}
|
||||
for _, m := range f.toasts {
|
||||
fmt.Fprintf(&b, " %s to %d tap acknowledged %q\n", m.At.Format("15:04:05.000"), m.Chat, firstLine(m.Text))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// button is the data of the button labelled so on a message.
|
||||
func button(m tgMessage, label string) string {
|
||||
for _, b := range m.Buttons {
|
||||
if b[0] == label {
|
||||
return b[1]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func labels(m tgMessage) []string {
|
||||
var out []string
|
||||
for _, b := range m.Buttons {
|
||||
out = append(out, b[0])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buttonsOf(markup any) [][2]string {
|
||||
var out [][2]string
|
||||
m, _ := markup.(map[string]any)
|
||||
rows, _ := m["inline_keyboard"].([]any)
|
||||
for _, row := range rows {
|
||||
cells, _ := row.([]any)
|
||||
for _, c := range cells {
|
||||
cell, _ := c.(map[string]any)
|
||||
text, _ := cell["text"].(string)
|
||||
data, _ := cell["callback_data"].(string)
|
||||
out = append(out, [2]string{text, data})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func num(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case int64:
|
||||
return float64(n)
|
||||
case int:
|
||||
return float64(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func orEmpty(in []map[string]any) []map[string]any {
|
||||
if in == nil {
|
||||
return []map[string]any{}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return s[:i] + " …"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var htmlTag = regexp.MustCompile(`</?[a-zA-Z][^>]*>`)
|
||||
|
||||
// shownOfHTML is a text sent in Telegram's HTML as the service shows it and gives it back: its tags read and taken
|
||||
// out, its entities written as the characters they stand for.
|
||||
func shownOfHTML(text string) string {
|
||||
return html.UnescapeString(htmlTag.ReplaceAllString(text, ""))
|
||||
}
|
||||
|
||||
// The fake phone shows a message sent in HTML as Telegram does (novox/hq issue 692): the words without their
|
||||
// markup, so a tap carries back what the channel digests, as on the real service.
|
||||
func TestTheFakePhoneShowsHTMLAsTelegramDoes(t *testing.T) {
|
||||
in := "<b>Flip the lab's switch?</b>\nA <b> & c\n<code>+ k: v</code>\n\n<blockquote expandable>Asked by lab-asker.</blockquote>"
|
||||
want := "Flip the lab's switch?\nA <b> & c\n+ k: v\n\nAsked by lab-asker."
|
||||
if got := shownOfHTML(in); got != want {
|
||||
t.Errorf("%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
module github.com/novox/mesh-lab/asks
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.10
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.10 h1:EHhnPRrkw7/CSQAqRsxYnTvlTZ3dI02IPO6WSmVX80o=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.10/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,537 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// machine is the lab's one machine, as the controller's fixture names it.
|
||||
const machine = "anchor"
|
||||
|
||||
// credential is what a lab process connects as (the controller fixture's credentials.json, mesh-tools'
|
||||
// bus.Credential).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// lab is one run of the proof: the repositories it builds from, its directory, its bus, its phone.
|
||||
type lab struct {
|
||||
t *testing.T
|
||||
repos clones
|
||||
dir string
|
||||
url string
|
||||
creds map[string]credential
|
||||
server *server.Server
|
||||
// observer is the lab's own user on the bus, outside the mesh's composition: it plays the controller's
|
||||
// conditions verb and reads the router's state, and is never used to show that something is allowed.
|
||||
observer *nats.Conn
|
||||
tg *fakeTelegram
|
||||
|
||||
mu sync.Mutex
|
||||
conditions []map[string]any
|
||||
// notFree are the machines the controller's root-free verb answers as not root-free.
|
||||
notFree map[string]string
|
||||
rootAsked int
|
||||
desk []string // what the desk was shown
|
||||
procs []*exec.Cmd
|
||||
logs map[string]*bytes.Buffer
|
||||
}
|
||||
|
||||
// clones is where each core repository the proof builds is checked out: side by side in MESH_LAB_REPOS (beside
|
||||
// mesh-lab by default), or one by one in MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and MESH_LAB_CATALOGUE — a branch
|
||||
// under review is often a worktree of its own.
|
||||
type clones struct{ controller, tools, catalogue string }
|
||||
|
||||
func clonesFor(t *testing.T) clones {
|
||||
repos := os.Getenv("MESH_LAB_REPOS")
|
||||
if repos == "" {
|
||||
repos = filepath.Join("..", "..")
|
||||
}
|
||||
at := func(env, name string) string {
|
||||
dir := os.Getenv(env)
|
||||
if dir == "" {
|
||||
dir = filepath.Join(repos, name)
|
||||
}
|
||||
abs, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return abs
|
||||
}
|
||||
c := clones{controller: at("MESH_LAB_CONTROLLER", "mesh-controller"), tools: at("MESH_LAB_TOOLS", "mesh-tools"),
|
||||
catalogue: at("MESH_LAB_CATALOGUE", "mesh-catalog")}
|
||||
for _, want := range []string{filepath.Join(c.controller, "go.mod"), filepath.Join(c.tools, "node-tools", "go.mod"),
|
||||
filepath.Join(c.catalogue, "modules", "messenger", "go.mod"), filepath.Join(c.catalogue, "modules", "telegram", "go.mod")} {
|
||||
if _, err := os.Stat(want); err != nil {
|
||||
t.Skipf("NOT RUN: the proof builds the controller, the runtime, the router and the Telegram channel from "+
|
||||
"their clones, and there is no %s (set MESH_LAB_REPOS, or MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and "+
|
||||
"MESH_LAB_CATALOGUE)", want)
|
||||
}
|
||||
}
|
||||
// The controller beside is, on the build seat, the one the mesh runs: until it carries the proof's fixture
|
||||
// (mesh-controller's TestTheAsksLabBus), the proof cannot compose its bus, and says so rather than fail
|
||||
// every pull request of the lab for a merge not yet made.
|
||||
fixtures, _ := filepath.Glob(filepath.Join(c.controller, "internal", "inventory", "*_test.go"))
|
||||
has := false
|
||||
for _, f := range fixtures {
|
||||
if body, err := os.ReadFile(f); err == nil && strings.Contains(string(body), "func TestTheAsksLabBus(") {
|
||||
has = true
|
||||
}
|
||||
}
|
||||
if !has {
|
||||
t.Skipf("NOT RUN: the controller at %s has no TestTheAsksLabBus, the fixture that composes the proof's bus "+
|
||||
"(mesh-controller #157): the proof runs once the controller it is given carries it", c.controller)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// run runs a command in a directory and fails the test with what it said.
|
||||
func (l *lab) run(dir string, env []string, name string, args ...string) string {
|
||||
l.t.Helper()
|
||||
cmd := exec.Command(name, args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
l.t.Fatalf("%s %s in %s: %v\n%s", name, strings.Join(args, " "), dir, err, out)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// compose asks the controller at its commit for the bus it would compose, then (bus set) to raise it.
|
||||
func (l *lab) compose(bus string) {
|
||||
// The anchor composed as a machine the controller measured root-free: verified-sender reaches the router
|
||||
// only then (ADR 0259 §8). The not-free case is played live, through the root-free verb, in the proof.
|
||||
env := []string{"GOFLAGS=-mod=vendor", "GOPROXY=off", "MESH_LAB_ASKS_OUT=" + l.dir, "MESH_LAB_ASKS_ROOT_FREE=true",
|
||||
"MESH_LAB_ASKS_CATALOGUE=" + filepath.Join(l.repos.catalogue, "modules"),
|
||||
"MESH_LAB_ASKS_RUNTIME=" + filepath.Join(l.repos.tools, "node-tools", "module.json")}
|
||||
if bus != "" {
|
||||
env = append(env, "MESH_LAB_ASKS_BUS="+bus)
|
||||
}
|
||||
out := l.run(l.repos.controller, env, "go", "test", "-count=1", "-v", "-run",
|
||||
"^TestTheAsksLabBus$", "./internal/inventory/")
|
||||
if !strings.Contains(out, "--- PASS: TestTheAsksLabBus") {
|
||||
l.t.Fatalf("the controller did not compose the lab's bus (is its fixture on this commit?):\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// startBus starts a server of the mesh's release with the controller's composed accounts, and the lab's
|
||||
// observer beside them.
|
||||
func (l *lab) startBus() {
|
||||
l.t.Helper()
|
||||
accounts, err := os.ReadFile(filepath.Join(l.dir, "accounts.conf"))
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
const usersOpen = " users = [\n"
|
||||
if !bytes.Contains(accounts, []byte(usersOpen)) {
|
||||
l.t.Fatalf("the composed accounts block has no users list:\n%s", accounts)
|
||||
}
|
||||
observerPassword := fmt.Sprintf("observer-%d", time.Now().UnixNano())
|
||||
accounts = bytes.Replace(accounts, []byte(usersOpen),
|
||||
[]byte(usersOpen+fmt.Sprintf(" { user: \"lab-observer\", password: %q }\n", observerPassword)), 1)
|
||||
port := freePort(l.t)
|
||||
conf := fmt.Sprintf("listen: \"127.0.0.1:%d\"\njetstream {\n store_dir: %q\n}\n\n%s", port,
|
||||
filepath.Join(l.dir, "jetstream"), accounts)
|
||||
path := filepath.Join(l.dir, "bus.conf")
|
||||
if err := os.WriteFile(path, []byte(conf), 0o600); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(path)
|
||||
if err != nil {
|
||||
l.t.Fatalf("the composed configuration is not one the server reads: %v", err)
|
||||
}
|
||||
// The server's own log, in the lab's directory: a refusal it says is evidence, kept with the run.
|
||||
opts.NoSigs, opts.LogFile = true, filepath.Join(l.dir, "bus.log")
|
||||
opts.Trace = os.Getenv("MESH_LAB_ASKS_TRACE") != ""
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
s.ConfigureLogger()
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
l.t.Fatal("the lab's bus did not come up")
|
||||
}
|
||||
l.server, l.url = s, s.ClientURL()
|
||||
l.t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
|
||||
raw, err := os.ReadFile(filepath.Join(l.dir, "credentials.json"))
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &l.creds); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.observer, err = nats.Connect(l.url, nats.UserInfo("lab-observer", observerPassword))
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(l.observer.Close)
|
||||
}
|
||||
|
||||
func freePort(t *testing.T) int {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
return ln.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// as connects as one of the composed users. refused collects what the server refused it.
|
||||
func (l *lab) as(user string) (*nats.Conn, *refusals) {
|
||||
l.t.Helper()
|
||||
c, ok := l.creds[user]
|
||||
if !ok {
|
||||
l.t.Fatalf("the controller composed no user %s", user)
|
||||
}
|
||||
r := &refusals{}
|
||||
nc, err := nats.Connect(l.url, nats.UserInfo(c.User, c.Password), nats.CustomInboxPrefix("_INBOX."+c.User),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { r.add(err) }))
|
||||
if err != nil {
|
||||
l.t.Fatalf("connecting as %s: %v", user, err)
|
||||
}
|
||||
l.t.Cleanup(nc.Close)
|
||||
return nc, r
|
||||
}
|
||||
|
||||
// refusals are the permission violations the server said to one connection.
|
||||
type refusals struct {
|
||||
mu sync.Mutex
|
||||
said []string
|
||||
}
|
||||
|
||||
func (r *refusals) add(err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.said = append(r.said, err.Error())
|
||||
}
|
||||
|
||||
func (r *refusals) of(subject string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, s := range r.said {
|
||||
if strings.Contains(strings.ToLower(s), "permissions violation") && strings.Contains(s, subject) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// refusedToPublish publishes once as a user and says whether the server refused it.
|
||||
func (l *lab) refusedToPublish(user, subject string, body []byte) bool {
|
||||
l.t.Helper()
|
||||
nc, r := l.as(user)
|
||||
if err := nc.Publish(subject, body); err != nil {
|
||||
return true
|
||||
}
|
||||
_ = nc.Flush()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if r.of(subject) {
|
||||
return true
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// build builds one Go program from a directory into the lab's bin.
|
||||
func (l *lab) build(dir, pkg, name string, ldflags string) string {
|
||||
out := filepath.Join(l.dir, "bin", name)
|
||||
args := []string{"build", "-o", out}
|
||||
if ldflags != "" {
|
||||
args = append(args, "-ldflags", ldflags)
|
||||
}
|
||||
l.run(dir, []string{"GOPRIVATE=git.novox.be", "GOFLAGS=-mod=mod", "CGO_ENABLED=0"}, "go", append(args, pkg)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// runtime starts the mesh's runtime on a module's own credential, serving that module's bundle alone: how
|
||||
// the mesh runs a module that `runs-as` an account of its own.
|
||||
func (l *lab) runtime(module, runtimeBin, bundle string, env map[string]string) {
|
||||
l.t.Helper()
|
||||
user := machine + "." + module
|
||||
c := l.creds[user]
|
||||
c.URL, c.Node, c.Module = l.url, machine, module
|
||||
credPath := filepath.Join(l.dir, module, "broker")
|
||||
if err := os.MkdirAll(filepath.Dir(credPath), 0o700); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
if err := os.WriteFile(credPath, raw, 0o600); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
toolEnv, _ := json.Marshal(map[string]map[string]string{module: env})
|
||||
cmd := exec.Command(runtimeBin, "serve")
|
||||
cmd.Env = append(os.Environ(), "MESH_BROKER_FILE="+credPath, "MESH_TOOL_MODULES="+module+"="+bundle,
|
||||
"MESH_TOOL_ENV="+string(toolEnv))
|
||||
logs := &bytes.Buffer{}
|
||||
cmd.Stdout, cmd.Stderr = &lockedWriter{w: logs, mu: &l.mu}, &lockedWriter{w: logs, mu: &l.mu}
|
||||
if err := cmd.Start(); err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.procs = append(l.procs, cmd)
|
||||
if l.logs == nil {
|
||||
l.logs = map[string]*bytes.Buffer{}
|
||||
}
|
||||
l.logs[module] = logs
|
||||
l.mu.Unlock()
|
||||
l.t.Cleanup(func() {
|
||||
_ = cmd.Process.Signal(os.Interrupt)
|
||||
done := make(chan struct{})
|
||||
go func() { _ = cmd.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
if l.t.Failed() || os.Getenv("MESH_LAB_ASKS_SAY") != "" {
|
||||
l.mu.Lock()
|
||||
l.t.Logf("--- what the %s runtime said:\n%s", module, logs.String())
|
||||
l.mu.Unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// said is what a runtime said so far.
|
||||
func (l *lab) said(module string) string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if b := l.logs[module]; b != nil {
|
||||
return b.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// waitSaid waits for a runtime to say something.
|
||||
func (l *lab) waitSaid(module, what string, within time.Duration) {
|
||||
l.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
if strings.Contains(l.said(module), what) {
|
||||
return
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
l.t.Fatalf("the %s runtime never said %q within %s:\n%s", module, what, within, l.said(module))
|
||||
}
|
||||
|
||||
type lockedWriter struct {
|
||||
w *bytes.Buffer
|
||||
mu *sync.Mutex
|
||||
}
|
||||
|
||||
func (w *lockedWriter) Write(p []byte) (int, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.w.Write(p)
|
||||
}
|
||||
|
||||
// playController answers the controller's `conditions` verb from the lab's list, and its `root-free` verb from
|
||||
// the lab's word on each machine (free unless notFree names it), as the router asks them. That a machine is
|
||||
// root-free is the controller's own judgement, proven in its repository (probe_agent_root_test.go); here the
|
||||
// lab says it, so the proof can show what the router does with each answer.
|
||||
func (l *lab) playController() {
|
||||
l.t.Helper()
|
||||
free, err := l.observer.Subscribe("mesh.seat.mesh-controller.tool.root-free", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
Machines []string `json:"machines"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
var out []map[string]any
|
||||
l.mu.Lock()
|
||||
for _, name := range args.Machines {
|
||||
why, not := l.notFree[name]
|
||||
if !not {
|
||||
why = "the lab says no agent can become root here"
|
||||
}
|
||||
out = append(out, map[string]any{"machine": name, "free": !not, "why": why,
|
||||
"judged": time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
}
|
||||
l.rootAsked++
|
||||
l.mu.Unlock()
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"machines": out}, "node": machine})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(func() { _ = free.Unsubscribe() })
|
||||
sub, err := l.observer.Subscribe("mesh.seat.mesh-controller.tool.conditions", func(m *nats.Msg) {
|
||||
l.mu.Lock()
|
||||
list := append([]map[string]any{}, l.conditions...)
|
||||
l.mu.Unlock()
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"conditions": list}, "node": machine})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
// raise says a condition as the controller does: on its event, and in its conditions verb from then on.
|
||||
func (l *lab) raise(c map[string]any) {
|
||||
l.t.Helper()
|
||||
l.mu.Lock()
|
||||
l.conditions = append(l.conditions, c)
|
||||
l.mu.Unlock()
|
||||
l.sayCondition("condition-raised", c)
|
||||
}
|
||||
|
||||
// clear ends a condition as the controller does.
|
||||
func (l *lab) clear(key string) {
|
||||
l.t.Helper()
|
||||
l.mu.Lock()
|
||||
var kept []map[string]any
|
||||
var gone map[string]any
|
||||
for _, c := range l.conditions {
|
||||
if c["key"] == key {
|
||||
gone = c
|
||||
continue
|
||||
}
|
||||
kept = append(kept, c)
|
||||
}
|
||||
l.conditions = kept
|
||||
l.mu.Unlock()
|
||||
if gone != nil {
|
||||
gone["cleared"] = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
l.sayCondition("condition-cleared", gone)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lab) sayCondition(event string, c map[string]any) {
|
||||
l.t.Helper()
|
||||
nc, _ := l.as("controller")
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
msg := nats.NewMsg("mesh.seat.mesh-controller.event." + event)
|
||||
msg.Data = raw
|
||||
msg.Header.Set("x-event-id", fmt.Sprintf("%s.%v.%d", event, c["key"], time.Now().UnixNano()))
|
||||
if _, err := js.PublishMsg(msg); err != nil {
|
||||
l.t.Fatalf("saying %s as the controller: %v", event, err)
|
||||
}
|
||||
}
|
||||
|
||||
// playDesk is the desk channel (kind `desktop`) at the bus, on the machine's runtime's credential, which
|
||||
// carries the desk channel: it says it is ready, and keeps what the router shows it — the link's code.
|
||||
func (l *lab) playDesk(ctx context.Context) {
|
||||
l.t.Helper()
|
||||
// The desk channel runs as an account of its own (it carries a link's code), on its own credential.
|
||||
nc, _ := l.as(machine + ".desk-channel")
|
||||
stand := func() {
|
||||
raw, _ := json.Marshal(map[string]any{"ready": true, "at": time.Now().UTC()})
|
||||
js, _ := nc.JetStream()
|
||||
msg := nats.NewMsg("mesh.seat.intake.event.standing.desktop")
|
||||
msg.Data = raw
|
||||
msg.Header.Set("x-event-id", fmt.Sprintf("standing.desktop.%d", time.Now().UnixNano()))
|
||||
if _, err := js.PublishMsg(msg); err != nil {
|
||||
l.t.Logf("the desk could not say it is ready: %v", err)
|
||||
}
|
||||
}
|
||||
stand()
|
||||
go func() {
|
||||
tick := time.NewTicker(20 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
stand()
|
||||
}
|
||||
}
|
||||
}()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
bind, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
worker, err := js.Consumer(bind, "SEAT_CHANNEL", "SEAT_CHANNEL_DESKTOP_worker")
|
||||
if err != nil {
|
||||
l.t.Fatalf("the desk's worker: %v", err)
|
||||
}
|
||||
consuming, err := worker.Consume(func(m jetstream.Msg) {
|
||||
var shown struct {
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data(), &shown)
|
||||
l.mu.Lock()
|
||||
l.desk = append(l.desk, shown.Title+": "+shown.Body)
|
||||
l.mu.Unlock()
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
l.t.Cleanup(consuming.Stop)
|
||||
}
|
||||
|
||||
var linkCode = regexp.MustCompile(`(\d{3}) (\d{3})\.`)
|
||||
|
||||
// codeOnTheDesk waits for the link's code the router shows on the desk.
|
||||
func (l *lab) codeOnTheDesk(within time.Duration) string {
|
||||
l.t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
l.mu.Lock()
|
||||
for _, s := range l.desk {
|
||||
if m := linkCode.FindStringSubmatch(s); m != nil {
|
||||
l.mu.Unlock()
|
||||
return m[1] + m[2]
|
||||
}
|
||||
}
|
||||
l.mu.Unlock()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.t.Fatalf("the desk was never shown a code; it was shown %q\nthe router said:\n%s", l.desk, l.logs["messenger"])
|
||||
return ""
|
||||
}
|
||||
|
||||
// routerRecord reads one key of the router's state, as the lab's observer.
|
||||
func (l *lab) routerRecord(bucket, key string) (jetstream.KeyValueEntry, jetstream.KeyValue) {
|
||||
l.t.Helper()
|
||||
js, err := jetstream.New(l.observer)
|
||||
if err != nil {
|
||||
l.t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
l.t.Fatalf("the router's bucket %s: %v", bucket, err)
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
l.t.Fatalf("the router's %s %s: %v", bucket, key, err)
|
||||
}
|
||||
return e, kv
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# The live acceptance of the operator's answers (novox/hq ADR 0259), after rollout, at the controller's
|
||||
# terminal on the control node. The lab's proof (TestTheOperatorsAnswerEndToEnd) ran the same flow with a fake
|
||||
# Telegram; this runs it once for real, with a question that changes nothing.
|
||||
#
|
||||
# sh live-acceptance.sh check, ask the rehearsal, wait for the answer, read the record
|
||||
#
|
||||
# It reads and asks only: no setting, assignment or push. The one thing it starts is a rehearsal, a question the
|
||||
# operator answers on the phone; approving it performs nothing and is recorded as the operator's decision.
|
||||
#
|
||||
# It stops at the first check that fails and says what to do. Exit 0 means: the question reached the phone
|
||||
# with Approve and Decline, the answer came back as a warrant, and the hand-act log names who answered,
|
||||
# through which channel, with which proofs.
|
||||
set -eu
|
||||
|
||||
# mesh-cli asks the controller; on the control node, from the operator's own account, a line is the
|
||||
# controller's terminal (ADR 0272). `nox` is its interactive alias; a script says mesh-cli.
|
||||
mc=${MESH_CLI:-mesh-cli}
|
||||
stop() { echo "NOT ACCEPTED: $*"; exit 1; }
|
||||
|
||||
echo "1. Nothing open says the router's or Telegram's machine is not root-free (root-not-free, agent-root)."
|
||||
if $mc conditions | grep -q 'root-not-free\|agent-root'; then
|
||||
$mc conditions | grep 'root-not-free\|agent-root'
|
||||
stop "while an agent can become root there, Telegram only acknowledges; close it first (ADR 0266, ADR 0268)"
|
||||
fi
|
||||
|
||||
echo "2. The Telegram channel holds a working bot token and its updates arrive."
|
||||
tg=$($mc ask telegram telegram_status) || stop "the telegram module does not answer"
|
||||
echo "$tg" | grep -q '"can_send": *true' || { echo "$tg"; stop "Telegram cannot send now (why_not above): give the bot token again at the controller's terminal (hq ADR 0259, the operator's steps), then push the control node"; }
|
||||
|
||||
echo "3. One Telegram account is linked as the operator, and its hour has passed."
|
||||
ids=$($mc ask messenger messenger_identities) || stop "the router does not answer"
|
||||
echo "$ids"
|
||||
echo "$ids" | grep -q '"telegram"' || stop "no Telegram account is linked: send /start to the bot, then the code from the desk"
|
||||
# The router answers approvals from a new link only an hour after it was made (ADR 0259 §7): said here, with the
|
||||
# time, rather than seen as a rehearsal nobody's approval reaches.
|
||||
from=$(echo "$ids" | tr -d '\n ' | grep -o '"kind":"telegram"[^}]*' | grep -o '"approve-from":"[^"]*"' | cut -d'"' -f4 | head -n 1)
|
||||
[ -n "$from" ] || stop "the router's word on the Telegram account says no time it can approve from"
|
||||
starts=$(date -d "$from" +%s 2>/dev/null) || stop "the time the Telegram account can approve from is not a time: $from"
|
||||
if [ "$(date +%s)" -lt "$starts" ]; then
|
||||
stop "the Telegram account was linked less than an hour ago — approvals start at $(date -d "$from" '+%H:%M') (local time); run this again then"
|
||||
fi
|
||||
|
||||
echo "4. The rehearsal: a question on your phone. Approve it there within 15 minutes."
|
||||
said=$($mc rehearse --for 15m) || stop "the rehearsal could not be asked"
|
||||
echo "$said"
|
||||
id=$(echo "$said" | sed -n 's/^rehearsal \([A-Za-z0-9_-]*\) asked.*/\1/p')
|
||||
[ -n "$id" ] || stop "the rehearsal did not say its id"
|
||||
|
||||
echo "5. Waiting for your answer in the hand-act log (every 10 s, at most 16 minutes)."
|
||||
i=0
|
||||
while [ $i -lt 96 ]; do
|
||||
acts=$($mc hand-acts --days 1 --json 2>/dev/null || true)
|
||||
mine=$(echo "$acts" | tr -d '\n' | grep -o "{[^{}]*\"ask\": *\"$id\"[^{}]*}" || true)
|
||||
if [ -n "$mine" ]; then
|
||||
echo "$mine"
|
||||
echo "$mine" | grep -q 'rehearsal=decline' && stop "the rehearsal was declined: run it again and choose Approve to see an approval recorded"
|
||||
echo "$mine" | grep -q 'rehearsal=approve' || stop "the rehearsal's record names no approval"
|
||||
echo "$mine" | grep -q '"via": *"telegram (telegram), user id verified"' || stop "the record does not say the answer came through Telegram from a verified account"
|
||||
echo "$mine" | grep -q '"P1"' || stop "the record does not carry the proof"
|
||||
echo "$mine" | grep -q '"outcome": *"done"' || stop "the rehearsal's act did not end done"
|
||||
echo "ACCEPTED: the rehearsal was approved on Telegram, acted on once, and recorded as your decision."
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 10
|
||||
done
|
||||
stop "no answer was recorded within 16 minutes: check messenger_status and telegram_status"
|
||||
@@ -0,0 +1,17 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// live-acceptance.sh is run by a person at the terminal and by no test: its syntax is judged here, as the lab's script
|
||||
// judged it with `sh -n` (novox/hq ADR 0529).
|
||||
func TestTheLiveAcceptanceScriptIsValidShell(t *testing.T) {
|
||||
if _, err := exec.LookPath("sh"); err != nil {
|
||||
t.Skip("NOT TESTED HERE: no sh on the path")
|
||||
}
|
||||
if out, err := exec.Command("sh", "-n", "live-acceptance.sh").CombinedOutput(); err != nil {
|
||||
t.Fatalf("live-acceptance.sh is not valid shell: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
//go:build integration
|
||||
|
||||
package asks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The operator's own Telegram account in the lab, and somebody else's.
|
||||
const (
|
||||
operatorAccount = 42
|
||||
strangerAccount = 99
|
||||
groupChat = -1001
|
||||
)
|
||||
|
||||
// TestTheOperatorsAnswerEndToEnd is the operator's acceptance of novox/hq ADR 0259, run before anybody uses it:
|
||||
//
|
||||
// "I get a Telegram message from the mesh with what is asked, why, by whom, and Approve/Decline buttons;
|
||||
// tapping Approve makes the mesh do exactly that one thing once; Decline or no answer does nothing;
|
||||
// nobody else (no agent, no other Telegram user, no replay of an old message) can produce an approval;
|
||||
// and I can see in the record who approved what."
|
||||
//
|
||||
// What is real: the bus, of the release the mesh runs, with the accounts the controller at its commit composes
|
||||
// for one machine and raised by that controller's own code (its fixture TestTheAsksLabBus); the mesh's runtime
|
||||
// (mesh-tools node-tools) serving the router (messenger) and the Telegram channel (telegram), each on its own
|
||||
// credential, as `runs-as` places them; the router's and the channel's code at their commits; the SDK's asker.
|
||||
//
|
||||
// What is the lab's: Telegram itself (a fake Bot API, the operator's phone), the desk (the desktop kind at the
|
||||
// bus, which shows the link's code), the controller's conditions verb (a list the lab keeps), and the asker
|
||||
// (the module `lab-asker`, the SDK's client in this process). The controller as an asker — that it acts on a
|
||||
// warrant once, through the verb the action names, and records it — is its own repository's test
|
||||
// (asker_test.go); the live acceptance after rollout runs it for real.
|
||||
func TestTheOperatorsAnswerEndToEnd(t *testing.T) {
|
||||
repos := clonesFor(t)
|
||||
dir := os.Getenv("MESH_LAB_ASKS_KEEP")
|
||||
if dir == "" {
|
||||
dir = t.TempDir()
|
||||
} else if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l := &lab{t: t, repos: repos, dir: dir}
|
||||
token := "123456789:" + strings.Repeat("A", 35)
|
||||
l.tg = newFakeTelegram(t, token)
|
||||
|
||||
// The bus, as the controller composes and raises it.
|
||||
l.compose("")
|
||||
l.startBus()
|
||||
l.compose(l.url)
|
||||
l.playController()
|
||||
|
||||
// The programs, at their commits. The channel is built to call the lab's phone in place of Telegram.
|
||||
runtimeBin := l.build(filepath.Join(repos.tools, "node-tools"), "./cmd/node-tools", "node-tools", "")
|
||||
routerBin := l.build(filepath.Join(repos.catalogue, "modules", "messenger"), "./cmd/messenger", "messenger", "")
|
||||
channelBin := l.build(filepath.Join(repos.catalogue, "modules", "telegram"), "./cmd/telegram", "telegram",
|
||||
"-X main.API="+l.tg.URL())
|
||||
|
||||
routerState := filepath.Join(dir, "messenger")
|
||||
if err := os.MkdirAll(routerState, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(routerState, "settings.json"), []byte(`{"time-zone": "UTC"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tokenFile := filepath.Join(dir, "telegram", "telegram-token")
|
||||
if err := os.MkdirAll(filepath.Dir(tokenFile), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(tokenFile, []byte(token+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
t.Cleanup(stop)
|
||||
l.playDesk(ctx)
|
||||
l.runtime("messenger", runtimeBin, routerBin, map[string]string{
|
||||
"MESH_MESSENGER_SETTINGS": filepath.Join(routerState, "settings.json"), "MESH_MESSENGER_STATE": routerState})
|
||||
l.runtime("telegram", runtimeBin, channelBin, map[string]string{"MESH_TELEGRAM_TOKEN_FILE": tokenFile})
|
||||
l.waitSaid("messenger", "taking the asks submitted", time.Minute)
|
||||
l.waitSaid("telegram", "taking the router's messages for this kind", time.Minute)
|
||||
a := newLabAsker(t, l)
|
||||
|
||||
// --- 1. The operator links their account: /start on the phone, the code from the desk.
|
||||
t.Run("the operator links their account with the code shown on the desk", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
l.tg.typed(operatorAccount, operatorAccount, "private", "/start")
|
||||
code := l.codeOnTheDesk(90 * time.Second)
|
||||
l.tg.typed(operatorAccount, operatorAccount, "private", code[:3]+" "+code[3:])
|
||||
l.tg.waitFor("the link said done", operatorAccount, since, 90*time.Second, func(m tgMessage) bool {
|
||||
return strings.Contains(m.Text, "Linked")
|
||||
})
|
||||
})
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
|
||||
// --- 2. Within the hour after a link, an approval is refused; the button is not spent.
|
||||
var first asks.Ask
|
||||
var firstMessage tgMessage
|
||||
t.Run("an approval in the hour after linking is refused and nothing is done", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
first = a.ask("a1", "Flip the lab's switch?", time.Hour)
|
||||
firstMessage = l.tg.waitFor("the first question with its buttons", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" && button(m, "Decline") != "" })
|
||||
for _, says := range []string{"Flip the lab's switch?", "approve or decline", "lab-asker",
|
||||
"the switch is flipped, once", "nothing is flipped"} {
|
||||
if !strings.Contains(firstMessage.Text, says) {
|
||||
t.Errorf("the question does not say %q (what is asked, why, by whom, what each answer does, what "+
|
||||
"no answer does): %q", says, firstMessage.Text)
|
||||
}
|
||||
}
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", firstMessage.ID, button(firstMessage, "Approve"))
|
||||
l.tg.waitFor("the refusal of an approval inside the hour", operatorAccount, tapped, 90*time.Second,
|
||||
func(m tgMessage) bool {
|
||||
return strings.Contains(m.Text, "That answer was not taken") && strings.Contains(m.Text, "linked at")
|
||||
})
|
||||
if !a.noWordOn("a1", 2*time.Second) || len(a.performed()) != 0 {
|
||||
t.Fatalf("an approval inside the hour was a warrant: %v", a.performed())
|
||||
}
|
||||
})
|
||||
|
||||
// The hour passes: the lab ages the link in the router's own record (the hour itself is the router's
|
||||
// unit test: an approve at 59 minutes refused, at 60 taken).
|
||||
t.Run("the hour after the link passes", func(t *testing.T) {
|
||||
e, kv := l.routerRecord("messenger_identities", fmt.Sprintf("telegram.%d", operatorAccount))
|
||||
var id map[string]any
|
||||
if err := json.Unmarshal(e.Value(), &id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
past := time.Now().Add(-61 * time.Minute).UTC()
|
||||
id["linked"], id["approve-from"] = past.Format(time.RFC3339Nano), past.Add(time.Hour).Format(time.RFC3339Nano)
|
||||
raw, _ := json.Marshal(id)
|
||||
if _, err := kv.Update(context.Background(), e.Key(), raw, e.Revision()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 3. Approve: the warrant, for this ask alone, performed once.
|
||||
t.Run("approve does exactly the one thing, once, and says who approved", func(t *testing.T) {
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", firstMessage.ID, button(firstMessage, "Approve"))
|
||||
w := a.warrantFor("a1", 30*time.Second)
|
||||
if w.Outcome != asks.OutcomeChosen || w.Option != "approve" || w.Level != asks.Approve {
|
||||
t.Fatalf("the warrant: %+v", w)
|
||||
}
|
||||
if w.By == nil || w.By.Kind != "telegram" || w.By.Identity != fmt.Sprint(operatorAccount) ||
|
||||
w.By.Verified != "user id verified" || w.Channel != "telegram" || strings.Join(w.Proofs, ",") != "P1" {
|
||||
t.Errorf("the warrant does not say who approved, how and where: %+v by %+v", w, w.By)
|
||||
}
|
||||
if w.AskDigest != first.Digest() {
|
||||
t.Errorf("the warrant names the digest %s, not the ask's %s", w.AskDigest, first.Digest())
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 || got[0]["arg.switch"] != "approve" {
|
||||
t.Fatalf("performed %v, want the one bound act once", got)
|
||||
}
|
||||
// Every copy says how it ended, its buttons gone.
|
||||
l.tg.waitFor("the question edited to its outcome", operatorAccount, tapped, 90*time.Second,
|
||||
func(m tgMessage) bool { return m.Edited && m.ID == firstMessage.ID && len(m.Buttons) == 0 })
|
||||
// And the router's record holds who approved what.
|
||||
e, _ := l.routerRecord("messenger_asks", "lab-asker.a1")
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
_ = json.Unmarshal(e.Value(), &rec)
|
||||
if rec.State != "chosen" || rec.Warrant == nil || rec.Warrant.By == nil || rec.Warrant.By.Identity != fmt.Sprint(operatorAccount) {
|
||||
t.Errorf("the router's record: %s", e.Value())
|
||||
}
|
||||
t.Logf("the record reads: %s", w.Says())
|
||||
})
|
||||
|
||||
// --- 4. A replayed tap, and a redelivered warrant, do nothing more.
|
||||
// The replay is the operator's own answer again: the router says it is the one already recorded (on the tap's
|
||||
// acknowledgement, issue 372), not refused as if the approval failed, and it is no second warrant or act.
|
||||
t.Run("a replayed tap and a redelivered warrant do nothing more", func(t *testing.T) {
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", firstMessage.ID, button(firstMessage, "Approve"))
|
||||
l.tg.toldTapper("that a replayed tap is the answer already recorded", operatorAccount, tapped, 90*time.Second,
|
||||
func(text string) bool {
|
||||
return strings.Contains(text, "You already chose Approve") && strings.Contains(text, "recorded")
|
||||
})
|
||||
time.Sleep(2 * time.Second)
|
||||
if n := a.wordsOn("a1"); n != 1 {
|
||||
t.Errorf("the router said %d words on one ask", n)
|
||||
}
|
||||
w := a.warrantFor("a1", time.Second)
|
||||
if h := a.take(w); h != asker.Done {
|
||||
t.Errorf("a warrant heard again was %s", h)
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("performed %v after a replay", got)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 5. Somebody else's account, and a group, cannot answer; Decline does nothing.
|
||||
t.Run("another account and a group cannot answer, and decline does nothing", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
a.ask("a2", "Flip it once more?", time.Hour)
|
||||
m := l.tg.waitFor("the second question", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" })
|
||||
// The buttons' data reached a stranger (a forwarded question), who taps in their own chat with the bot:
|
||||
// the channel never showed that message there, so the tap goes nowhere, and the stranger is told
|
||||
// nothing. (A stranger's answer that does reach the router is refused there and told to the operator by
|
||||
// name: the router's own test.)
|
||||
l.tg.tapped(strangerAccount, strangerAccount, "private", m.ID, button(m, "Approve"))
|
||||
// The operator taps a message whose words are not the ones the channel put there.
|
||||
l.tg.tappedOn(operatorAccount, operatorAccount, "private", m.ID, button(m, "Approve"), "Flip ALL the switches?")
|
||||
// The operator taps in a group the bot is in: not their own chat, dropped by the channel.
|
||||
l.tg.tapped(operatorAccount, groupChat, "supergroup", m.ID, button(m, "Approve"))
|
||||
if !a.noWordOn("a2", 3*time.Second) {
|
||||
t.Fatal("a tap from another account, on changed words, or from a group, was a warrant")
|
||||
}
|
||||
if got := l.tg.since(strangerAccount, since); len(got) != 0 {
|
||||
t.Errorf("the bot answered the stranger: %+v", got)
|
||||
}
|
||||
if got := l.tg.since(groupChat, since); len(got) != 0 {
|
||||
t.Errorf("the bot answered in a group: %+v", got)
|
||||
}
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", m.ID, button(m, "Decline"))
|
||||
w := a.warrantFor("a2", 30*time.Second)
|
||||
if w.Option != "decline" || w.Level != asks.Acknowledge {
|
||||
t.Errorf("decline's warrant: %+v", w)
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("decline performed something: %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 6. No agent can produce an approval: the bus refuses every forgery, as the controller composed it.
|
||||
t.Run("no agent, channel or module but the router can say a warrant, and none asks in another's name", func(t *testing.T) {
|
||||
warrant, _ := json.Marshal(asks.Warrant{Ask: "a9", Asker: "lab-asker", Outcome: asks.OutcomeChosen,
|
||||
Option: "approve", Level: asks.Approve, AskDigest: first.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}})
|
||||
decided := asks.DecidedSubject("lab-asker")
|
||||
// The control: what is granted is not refused, so a refusal below is the bus's word and not the lab's.
|
||||
// The channel's control is a true standing, now: a false one would hold the router's messages.
|
||||
standing, _ := json.Marshal(asks.Standing{Ready: true, EditsSilently: true, At: time.Now().UTC()})
|
||||
for _, try := range []struct {
|
||||
user, subject string
|
||||
body []byte
|
||||
}{
|
||||
{machine + ".lab-asker", "mesh.seat.operator-channel.accept.cancel.lab-asker", []byte(`{"id":"none"}`)},
|
||||
{machine + ".telegram", asks.IntakeSubject(asks.StandingWhat, "telegram"), standing},
|
||||
} {
|
||||
if l.refusedToPublish(try.user, try.subject, try.body) {
|
||||
t.Fatalf("the control failed: %s was refused %s, which the controller grants it", try.user, try.subject)
|
||||
}
|
||||
}
|
||||
for _, user := range []string{
|
||||
machine + ".node-tools", // the machine's runtime: every agent's tool calls go through one
|
||||
machine + ".lab-asker", // the asker itself
|
||||
machine + ".lab-bystander", // any other module
|
||||
machine + ".telegram", // the channel
|
||||
"node." + machine, // the machine's node-engine
|
||||
"controller", // the controller asks; it does not decide
|
||||
} {
|
||||
if !l.refusedToPublish(user, decided, warrant) {
|
||||
t.Errorf("%s may say a warrant on %s", user, decided)
|
||||
}
|
||||
}
|
||||
for _, try := range []struct{ user, subject, what string }{
|
||||
{machine + ".lab-asker", asks.AskSubject("mesh-controller"), "an ask in another asker's name"},
|
||||
{machine + ".lab-bystander", asks.AskSubject("lab-bystander"), "an ask from a module that may not ask"},
|
||||
{machine + ".node-tools", asks.AskSubject("mesh-controller"), "an agent asking as the controller through the runtime"},
|
||||
{machine + ".node-tools", asks.IntakeSubject(asks.Chosen, "telegram"), "an agent saying a Telegram tap"},
|
||||
{machine + ".lab-asker", asks.IntakeSubject(asks.Chosen, "telegram"), "an asker saying a Telegram tap"},
|
||||
{machine + ".node-tools", "$KV.messenger_identities.telegram.666", "an agent linking an account of its own"},
|
||||
{machine + ".lab-asker", "$KV.messenger_asks.lab-asker.a1", "an asker rewriting the router's record"},
|
||||
{machine + ".telegram", asks.ProofSubject(asks.CodeProof, "desktop"), "a channel speaking for another kind"},
|
||||
} {
|
||||
if !l.refusedToPublish(try.user, try.subject, []byte(`{}`)) {
|
||||
t.Errorf("%s: %s may publish %s", try.what, try.user, try.subject)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if n := a.wordsOn("a9"); n != 0 || len(a.performed()) != 1 {
|
||||
t.Fatalf("a forged warrant reached the asker: %d words, performed %v", n, a.performed())
|
||||
}
|
||||
})
|
||||
|
||||
// --- 7. While an agent can become root where the router runs, nothing proven there approves.
|
||||
t.Run("while the controller cannot say root is free where the router runs, nothing approves", func(t *testing.T) {
|
||||
// An approval the operator already sees on the phone, then root stops being free.
|
||||
since := time.Now()
|
||||
a.ask("a5", "Flip it while root is checked?", time.Hour)
|
||||
m := l.tg.waitFor("the question asked while root was free", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" })
|
||||
l.mu.Lock()
|
||||
l.notFree = map[string]string{machine: "an agent can become root without a person"}
|
||||
asked := l.rootAsked
|
||||
l.mu.Unlock()
|
||||
tapped := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", m.ID, button(m, "Approve"))
|
||||
l.tg.waitFor("the refusal of an approval while root is not free", operatorAccount, tapped, 90*time.Second,
|
||||
func(m tgMessage) bool { return strings.Contains(m.Text, "That answer was not taken") })
|
||||
l.mu.Lock()
|
||||
askedAgain := l.rootAsked > asked
|
||||
l.mu.Unlock()
|
||||
if !askedAgain {
|
||||
t.Error("the router judged an approval without asking the controller whether root is free")
|
||||
}
|
||||
if !a.noWordOn("a5", 2*time.Second) {
|
||||
t.Fatal("an approval was a warrant while root was not free")
|
||||
}
|
||||
// A new ask that needs an approval is refused to its asker, in words: no channel can carry it now.
|
||||
a.ask("a3", "Flip it while root is not free?", time.Hour)
|
||||
w := a.warrantFor("a3", 30*time.Second)
|
||||
if w.Outcome != asks.OutcomeRefused || w.By != nil {
|
||||
t.Errorf("an approving ask while root is not free: %+v", w)
|
||||
}
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("performed %v while root was not free", got)
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.notFree = nil
|
||||
l.mu.Unlock()
|
||||
if err := a.client.Cancel("a5"); err != nil {
|
||||
t.Errorf("taking the ask back: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// --- 8. No answer does nothing: the ask expires.
|
||||
t.Run("an ask nobody answers expires and nothing is done", func(t *testing.T) {
|
||||
since := time.Now()
|
||||
a.ask("a4", "Flip it if you answer in time?", 20*time.Second)
|
||||
m := l.tg.waitFor("the fourth question", operatorAccount, since, 90*time.Second,
|
||||
func(m tgMessage) bool { return !m.Edited && button(m, "Approve") != "" })
|
||||
w := a.warrantFor("a4", 2*time.Minute)
|
||||
if w.Outcome != asks.OutcomeExpired || w.By != nil {
|
||||
t.Errorf("the end of an unanswered ask: %+v", w)
|
||||
}
|
||||
late := time.Now()
|
||||
l.tg.tapped(operatorAccount, operatorAccount, "private", m.ID, button(m, "Approve"))
|
||||
l.tg.waitFor("the refusal of a late tap", operatorAccount, late, 90*time.Second,
|
||||
func(m tgMessage) bool { return strings.Contains(m.Text, "That answer was not taken") })
|
||||
time.Sleep(2 * time.Second)
|
||||
if got := a.performed(); len(got) != 1 {
|
||||
t.Fatalf("an expired ask performed something: %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Logf("performed, over the whole run: %v (one approval, one act)", a.performed())
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json",
|
||||
"pretest": "npm run typecheck",
|
||||
"test": "node --test --experimental-strip-types 'test/*.test.ts'",
|
||||
"test:integration": "node --experimental-strip-types src/cli.ts suite",
|
||||
"check": "npm run typecheck && npm test && npm run test:integration",
|
||||
|
||||
+410
@@ -0,0 +1,410 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The bed (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): **a declaration is proved before it is trusted.**
|
||||
//
|
||||
// Two of the nineteen image checks the catalogue's containers ship read *unhealthy* while working in the
|
||||
// mesh's hands — the studio's asked an address its program does not bind, the flow editor's read settings
|
||||
// from a path the module mounted elsewhere. Read without proof, they would have put back two good builds.
|
||||
// So the catalogue's merge check starts every resource whose declared check or image the change touches,
|
||||
// **alone**, on a throwaway runtime, and requires its check to see the program within its grace.
|
||||
//
|
||||
// **What it proves is the check's wiring**: that it can see the program working — a property of the image
|
||||
// and the declaration, not of the mesh. A resource is started with what the module declares and nothing of
|
||||
// the mesh's: its literal environment, its arguments, its files where the module writes them in full, and an
|
||||
// empty directory for every other place it mounts; no secret, no binding, no provider. So:
|
||||
//
|
||||
// - a check naming a provision in `needs` gets no provider here, and must only reach the program — an
|
||||
// http answer of any status, a connect;
|
||||
// - a program that does not stay up alone — it needs what the mesh gives it — is said as not proved here,
|
||||
// and judged on the first machine's gate, never passed as proved and never failed;
|
||||
// - a program that stays up while its check does not see it **fails**: that is the studio's fault.
|
||||
//
|
||||
// A tool check and a unit's own readiness need the mesh and a machine; they are said, not proved.
|
||||
|
||||
// BedResource is one resource the bed proves: its module, its id, and the resource as the manifest says it.
|
||||
type BedResource struct {
|
||||
Module string
|
||||
ID string
|
||||
Resource map[string]any
|
||||
// Listens are the module's endpoints, by name: the container's own port for each.
|
||||
Listens map[string]int
|
||||
}
|
||||
|
||||
// BedVerdict is what the bed found of one resource.
|
||||
type BedVerdict struct {
|
||||
Proved bool
|
||||
// Failed says the program stayed up and its check did not see it: the change's fault.
|
||||
Failed bool
|
||||
Said string
|
||||
}
|
||||
|
||||
// BedLook is how often the bed looks: often, because it proves the wiring, not the timing.
|
||||
var BedLook = 2 * time.Second
|
||||
|
||||
// BedFloor is the least the bed waits for a check to pass, whatever the grace: a program needs a moment to
|
||||
// listen even when the module declared no grace.
|
||||
var BedFloor = 30 * time.Second
|
||||
|
||||
// ChangedHealth is every long-running container resource of the catalogue at root that declares `health`
|
||||
// and whose resource changed against base (a git ref in that checkout): its declaration, its image, or
|
||||
// anything else of it. A manifest new on this branch counts whole. Read through git; an error when the
|
||||
// base cannot be read.
|
||||
func ChangedHealth(root, base string, show func(ref, path string) ([]byte, error)) ([]BedResource, error) {
|
||||
paths, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BedResource
|
||||
for _, p := range paths {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
was := map[string]string{}
|
||||
if before, err := show(base, rel); err == nil {
|
||||
for _, r := range resourcesOf(before) {
|
||||
was[fmt.Sprint(r["id"])] = canonical(r)
|
||||
}
|
||||
}
|
||||
module, listens, resources := manifestParts(raw)
|
||||
for _, r := range resources {
|
||||
if _, declared := r["health"]; !declared || fmt.Sprint(r["type"]) != "container" || !stays(r) {
|
||||
continue
|
||||
}
|
||||
if was[fmt.Sprint(r["id"])] == canonical(r) {
|
||||
continue
|
||||
}
|
||||
out = append(out, BedResource{Module: module, ID: fmt.Sprint(r["id"]), Resource: r, Listens: listens})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Module+"."+out[a].ID < out[b].Module+"."+out[b].ID })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func stays(r map[string]any) bool {
|
||||
once, _ := r["run-once"].(bool)
|
||||
return !once && r["schedule"] == nil
|
||||
}
|
||||
|
||||
func canonical(r map[string]any) string {
|
||||
raw, _ := json.Marshal(r)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func resourcesOf(raw []byte) []map[string]any {
|
||||
_, _, rs := manifestParts(raw)
|
||||
return rs
|
||||
}
|
||||
|
||||
func manifestParts(raw []byte) (string, map[string]int, []map[string]any) {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Listens []struct {
|
||||
Name string `json:"name"`
|
||||
Port int `json:"port"`
|
||||
} `json:"listens"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &m)
|
||||
listens := map[string]int{}
|
||||
for _, l := range m.Listens {
|
||||
if l.Name != "" {
|
||||
listens[l.Name] = l.Port
|
||||
}
|
||||
}
|
||||
return m.Module, listens, m.Resources
|
||||
}
|
||||
|
||||
// placeholder is anything the mesh fills in on a machine: a value holding one is not the module's alone.
|
||||
var placeholder = regexp.MustCompile(`\$\{[^}]*\}`)
|
||||
|
||||
// Prove starts one resource alone on the runtime and looks at it with its declared check until the check
|
||||
// sees it, the program stops, or its grace (at least BedFloor) runs out. Everything it made is removed.
|
||||
func (d *Docker) Prove(ctx context.Context, r BedResource, files map[string]string) BedVerdict {
|
||||
h, _ := r.Resource["health"].(map[string]any)
|
||||
kind := fmt.Sprint(h["kind"])
|
||||
switch kind {
|
||||
case "tool", "unit":
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s: a %s check needs the mesh and a machine; judged on the first "+
|
||||
"machine's gate", r.Module, r.ID, kind)}
|
||||
}
|
||||
image, _ := r.Resource["image"].(string)
|
||||
if image == "" || placeholder.MatchString(image) {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s: its image is built by the mesh (%v); judged on the first machine's gate",
|
||||
r.Module, r.ID, r.Resource["artifact"])}
|
||||
}
|
||||
if err := d.Pull(ctx, image); err != nil {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s: its image could not be fetched here: %v", r.Module, r.ID, oneLine(err.Error()))}
|
||||
}
|
||||
scratch, err := os.MkdirTemp("", "mesh-bed-")
|
||||
if err != nil {
|
||||
return BedVerdict{Said: err.Error()}
|
||||
}
|
||||
defer os.RemoveAll(scratch)
|
||||
// Its literal environment, and every env-file the module writes in full.
|
||||
var env []string
|
||||
if e, ok := r.Resource["env"].(map[string]any); ok {
|
||||
for k, v := range e {
|
||||
if s := fmt.Sprint(v); !placeholder.MatchString(s) {
|
||||
env = append(env, k+"="+s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if list, ok := r.Resource["env-file"].([]any); ok {
|
||||
for _, f := range list {
|
||||
if content, ok := files[fmt.Sprint(f)]; ok {
|
||||
env = append(env, envLines(content)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(env)
|
||||
// Its mounts: a file the module writes in full, or an empty place of its own.
|
||||
var binds []string
|
||||
if list, ok := r.Resource["volumes"].([]any); ok {
|
||||
for i, v := range list {
|
||||
src, dst, rest := splitMount(fmt.Sprint(v))
|
||||
if dst == "" || !strings.HasPrefix(dst, "/") {
|
||||
continue
|
||||
}
|
||||
local := filepath.Join(scratch, "m"+strconv.Itoa(i))
|
||||
if content, ok := files[src]; ok {
|
||||
if err := os.WriteFile(local, []byte(content), 0o644); err != nil {
|
||||
continue
|
||||
}
|
||||
} else if err := os.MkdirAll(local, 0o777); err != nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Chmod(local, 0o777)
|
||||
binds = append(binds, local+":"+dst+rest)
|
||||
}
|
||||
}
|
||||
var args []string
|
||||
if list, ok := r.Resource["args"].([]any); ok {
|
||||
for _, a := range list {
|
||||
args = append(args, fmt.Sprint(a))
|
||||
}
|
||||
}
|
||||
network := fmt.Sprintf("mesh-bed-%d", time.Now().UnixNano())
|
||||
netID, err := d.Network(ctx, network)
|
||||
if err != nil {
|
||||
return BedVerdict{Said: "the bed's network could not be made: " + err.Error()}
|
||||
}
|
||||
defer d.RemoveNetwork(context.Background(), netID)
|
||||
|
||||
grace, _ := time.ParseDuration(fmt.Sprint(h["grace"]))
|
||||
if _, said := h["grace"]; !said {
|
||||
grace = 60 * time.Second
|
||||
}
|
||||
if grace < BedFloor {
|
||||
grace = BedFloor
|
||||
}
|
||||
config := map[string]any{"Image": image, "Env": env, "Labels": map[string]string{d.Label: "bed"}}
|
||||
if len(args) > 0 {
|
||||
config["Cmd"] = args
|
||||
}
|
||||
switch kind {
|
||||
case "exec":
|
||||
config["Healthcheck"] = map[string]any{"Test": []string{"CMD-SHELL", fmt.Sprint(h["command"])},
|
||||
"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1}
|
||||
case "runtime":
|
||||
// The image's own command, adopted by name: an empty Test keeps it.
|
||||
config["Healthcheck"] = map[string]any{"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1,
|
||||
"Retries": 1}
|
||||
}
|
||||
config["HostConfig"] = map[string]any{"Binds": binds, "NetworkMode": network}
|
||||
name := fmt.Sprintf("mesh-bed-%s-%s-%d", r.Module, r.ID, time.Now().UnixNano())
|
||||
var made struct{ ID string }
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/create?name="+name, config, &made); err != nil {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s could not be made here: %v", r.Module, r.ID, oneLine(err.Error()))}
|
||||
}
|
||||
defer d.Remove(context.Background(), made.ID)
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s could not be started here: %v", r.Module, r.ID, oneLine(err.Error()))}
|
||||
}
|
||||
|
||||
port := r.Listens[fmt.Sprint(h["endpoint"])]
|
||||
needs := fmt.Sprint(h["needs"]) != "" && h["needs"] != nil
|
||||
deadline := time.Now().Add(grace)
|
||||
last := "it was never looked at"
|
||||
for {
|
||||
var seen struct {
|
||||
State struct {
|
||||
Running bool
|
||||
Health *struct {
|
||||
Status string
|
||||
Log []struct{ Output string }
|
||||
}
|
||||
}
|
||||
NetworkSettings struct {
|
||||
Networks map[string]struct{ IPAddress string }
|
||||
}
|
||||
}
|
||||
if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil {
|
||||
return BedVerdict{Said: err.Error()}
|
||||
}
|
||||
if !seen.State.Running {
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s does not stay up alone — it needs what the mesh gives it — so its "+
|
||||
"check is judged on the first machine's gate: %s", r.Module, r.ID, oneLine(lastLine(d.Logs(ctx, made.ID))))}
|
||||
}
|
||||
ok := false
|
||||
switch kind {
|
||||
case "exec", "runtime":
|
||||
hs := seen.State.Health
|
||||
switch {
|
||||
case hs == nil:
|
||||
last = "the container carries no check: its image ships none to adopt"
|
||||
case hs.Status == "healthy":
|
||||
ok = true
|
||||
default:
|
||||
last = "the runtime says " + hs.Status
|
||||
if n := len(hs.Log); n > 0 {
|
||||
last += ": " + oneLine(hs.Log[n-1].Output)
|
||||
}
|
||||
}
|
||||
case "http", "tcp":
|
||||
address := ""
|
||||
for _, n := range seen.NetworkSettings.Networks {
|
||||
address = n.IPAddress
|
||||
}
|
||||
ok, last = look(ctx, kind, address, port, h, needs)
|
||||
default:
|
||||
return BedVerdict{Said: fmt.Sprintf("%s.%s declares a %s check the bed does not know", r.Module, r.ID, kind)}
|
||||
}
|
||||
if ok {
|
||||
return BedVerdict{Proved: true, Said: fmt.Sprintf("%s.%s: its %s check sees the program", r.Module, r.ID, kind)}
|
||||
}
|
||||
if time.Now().After(deadline) || ctx.Err() != nil {
|
||||
return BedVerdict{Failed: true, Said: fmt.Sprintf("%s.%s stays up and its %s check does not see it within %s: %s",
|
||||
r.Module, r.ID, kind, grace, last)}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(BedLook):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// look is one http or tcp look at the program, from outside it, as the node-engine makes it. A check that
|
||||
// needs a provider only has to reach the program: any answer, a connect.
|
||||
func look(ctx context.Context, kind, address string, port int, h map[string]any, needs bool) (bool, string) {
|
||||
if address == "" || port == 0 {
|
||||
return false, "it has no address on the bed's network yet"
|
||||
}
|
||||
at := net.JoinHostPort(address, strconv.Itoa(port))
|
||||
ctx, cancel := context.WithTimeout(ctx, BedLook)
|
||||
defer cancel()
|
||||
if kind == "tcp" {
|
||||
c, err := (&net.Dialer{}).DialContext(ctx, "tcp", at)
|
||||
if err != nil {
|
||||
return false, oneLine(err.Error())
|
||||
}
|
||||
c.Close()
|
||||
return true, ""
|
||||
}
|
||||
scheme, _ := h["scheme"].(string)
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
path, _ := h["path"].(string)
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+at+path, nil)
|
||||
res, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
Transport: insecure()}).Do(req)
|
||||
if err != nil {
|
||||
return false, oneLine(err.Error())
|
||||
}
|
||||
res.Body.Close()
|
||||
if needs {
|
||||
return true, ""
|
||||
}
|
||||
want, _ := h["status"].(float64)
|
||||
switch {
|
||||
case want != 0 && res.StatusCode != int(want):
|
||||
return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, int(want))
|
||||
case want == 0 && res.StatusCode >= 400:
|
||||
return false, fmt.Sprintf("answered %d", res.StatusCode)
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func splitMount(v string) (src, dst, rest string) {
|
||||
parts := strings.SplitN(v, ":", 3)
|
||||
if len(parts) < 2 {
|
||||
return "", "", ""
|
||||
}
|
||||
if len(parts) == 3 {
|
||||
rest = ":" + parts[2]
|
||||
}
|
||||
return parts[0], parts[1], rest
|
||||
}
|
||||
|
||||
func envLines(content string) []string {
|
||||
var out []string
|
||||
s := bufio.NewScanner(strings.NewReader(content))
|
||||
for s.Scan() {
|
||||
line := strings.TrimSpace(s.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") || placeholder.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return lines[len(lines)-1]
|
||||
}
|
||||
|
||||
// FilesOf is every file a module writes in full — its content declared, nothing the mesh fills in — by the
|
||||
// path it writes it at, as the module names it: what the bed gives a container that mounts or reads it.
|
||||
func FilesOf(manifest []byte) map[string]string {
|
||||
_, _, rs := manifestParts(manifest)
|
||||
out := map[string]string{}
|
||||
for _, r := range rs {
|
||||
if fmt.Sprint(r["type"]) != "file" {
|
||||
continue
|
||||
}
|
||||
path, _ := r["path"].(string)
|
||||
content, ok := r["content"].(string)
|
||||
if path == "" || !ok || placeholder.MatchString(content) {
|
||||
continue
|
||||
}
|
||||
out[path] = content
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// insecure is a transport that asks whether a program answers, not whom to trust.
|
||||
func insecure() *http.Transport {
|
||||
return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true}
|
||||
}
|
||||
|
||||
// oneLine is the first line of what was said, short.
|
||||
func oneLine(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
if len(line) > 300 {
|
||||
line = line[:300] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
//go:build integration
|
||||
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **R-studio — a check that asks an address the program does not bind fails the bed, not a machine**
|
||||
// (novox/hq ADR 0240 Phase D, done when). The hosted database suite's studio binds the address the runtime
|
||||
// puts in HOSTNAME, so it answered only on its network address while its image's own check asked localhost:
|
||||
// unhealthy for ever while working, until the module set HOSTNAME=0.0.0.0. Adopted by name without proof, it
|
||||
// would have put back a good build.
|
||||
//
|
||||
// The replay is that image in miniature: a web server that binds $HOSTNAME's address, and an image check that
|
||||
// asks localhost. Adopted as the module declared it before the fix, the bed fails it; with the fix, it proves
|
||||
// it. The image is built here and removed after, with everything the bed made.
|
||||
func TestBedFailsTheStudiosFalseUnhealthy(t *testing.T) {
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the studio is a container")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
|
||||
defer cancel()
|
||||
if err := docker.Pull(ctx, "busybox:1.36"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tag := fmt.Sprintf("mesh-replay-studio:%d", time.Now().UnixNano())
|
||||
if err := docker.Build(ctx, tag, StudioDockerfile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.RemoveImage(context.Background(), tag)
|
||||
was := BedFloor
|
||||
BedFloor = 15 * time.Second
|
||||
defer func() { BedFloor = was }()
|
||||
|
||||
studio := func(env map[string]any) BedResource {
|
||||
r := map[string]any{"id": "studio", "type": "container", "name": "supabase-studio", "image": tag,
|
||||
"health": map[string]any{"kind": "runtime", "grace": "10s"}}
|
||||
if env != nil {
|
||||
r["env"] = env
|
||||
}
|
||||
return BedResource{Module: "supabase", ID: "studio", Resource: r, Listens: map[string]int{}}
|
||||
}
|
||||
before := docker.Prove(ctx, studio(nil), nil)
|
||||
if !before.Failed || !strings.Contains(before.Said, "does not see it") {
|
||||
t.Fatalf("the studio's false unhealthy was not failed on the bed: %+v", before)
|
||||
}
|
||||
t.Logf("before the fix: %s", before.Said)
|
||||
after := docker.Prove(ctx, studio(map[string]any{"HOSTNAME": "0.0.0.0"}), nil)
|
||||
if !after.Proved {
|
||||
t.Fatalf("the studio with HOSTNAME=0.0.0.0 was not proved: %+v", after)
|
||||
}
|
||||
t.Logf("with the fix: %s", after.Said)
|
||||
}
|
||||
|
||||
// StudioDockerfile is the studio in miniature: it serves on the address HOSTNAME names, and its own check
|
||||
// asks localhost, as the studio's image does.
|
||||
const StudioDockerfile = `FROM busybox:1.36
|
||||
RUN mkdir -p /www && echo studio > /www/index.html
|
||||
HEALTHCHECK --interval=5s --timeout=2s --retries=2 CMD wget -q -O /dev/null http://localhost:3000/ || exit 1
|
||||
CMD addr=$(grep -w "$HOSTNAME" /etc/hosts | head -n 1 | cut -f 1); exec httpd -f -p "${addr:-$HOSTNAME}:3000" -h /www
|
||||
`
|
||||
@@ -0,0 +1,111 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The catalogue's bed** (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): every long-running resource whose
|
||||
// declared `health` or image the change touches is started alone and its check must see the program within
|
||||
// its grace. Run by every merge check of the catalogue on the build seat, with the change's catalogue at
|
||||
// MESH_REPLAY_CATALOGUE; by hand against the catalogue beside the lab. What changed is against
|
||||
// MESH_BED_BASE (origin/main by default) in that checkout.
|
||||
func TestBedProvesEveryChangedHealthCheck(t *testing.T) {
|
||||
root := orDefault(os.Getenv("MESH_REPLAY_CATALOGUE"), filepath.Join("..", "..", "mesh-catalog"))
|
||||
if _, err := os.Stat(filepath.Join(root, "modules")); err != nil {
|
||||
t.Skipf("no catalogue at %s (MESH_REPLAY_CATALOGUE names it)", root)
|
||||
}
|
||||
base := orDefault(os.Getenv("MESH_BED_BASE"), "origin/main")
|
||||
show := func(ref, path string) ([]byte, error) {
|
||||
return exec.Command("git", "-c", "safe.directory=*", "-C", root, "show", ref+":"+path).Output()
|
||||
}
|
||||
if _, err := exec.Command("git", "-c", "safe.directory=*", "-C", root, "rev-parse", "--verify", base).Output(); err != nil {
|
||||
// What changed cannot be told, so every declared check is proved: the bed never passes what it did
|
||||
// not look at.
|
||||
t.Logf("%s is not in the catalogue at %s (%v): every declared check is proved", base, root, err)
|
||||
}
|
||||
changed, err := ChangedHealth(root, base, show)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(changed) == 0 {
|
||||
t.Logf("no declared check or image of a long-running resource changed against %s: nothing to prove", base)
|
||||
return
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Fatalf("%d changed check(s) and no container runtime to prove them on", len(changed))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Minute)
|
||||
defer cancel()
|
||||
for _, r := range changed {
|
||||
manifest, _ := os.ReadFile(filepath.Join(root, "modules", r.Module, "module.json"))
|
||||
v := docker.Prove(ctx, r, FilesOf(manifest))
|
||||
switch {
|
||||
case v.Failed:
|
||||
t.Errorf("FAILS ON THE BED: %s", v.Said)
|
||||
case v.Proved:
|
||||
t.Logf("proved: %s", v.Said)
|
||||
default:
|
||||
t.Logf("NOT PROVED HERE: %s", v.Said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the bed proves is what the change touches: a long-running container whose declared check or image
|
||||
// changed, or that is new; never a step, never one left as it was.
|
||||
func TestTheBedProvesWhatTheChangeTouches(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
git := func(args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", root, "-c", "user.email=lab@example", "-c", "user.name=lab"}, args...)...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v %s", args, err, out)
|
||||
}
|
||||
}
|
||||
write := func(module, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Join(root, "modules", module), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "modules", module, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
git("init", "-q", "-b", "main")
|
||||
write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:a","health":{"kind":"runtime"}},
|
||||
{"id":"seed","type":"container","image":"web@sha256:a","run-once":true,"health":{"kind":"runtime"}}]}`)
|
||||
write("db", `{"module":"db","resources":[{"id":"server","type":"container","image":"db@sha256:a","health":{"kind":"runtime"}}]}`)
|
||||
git("add", "-A")
|
||||
git("commit", "-qm", "base")
|
||||
write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:b","health":{"kind":"runtime"}},
|
||||
{"id":"seed","type":"container","image":"web@sha256:b","run-once":true,"health":{"kind":"runtime"}}]}`)
|
||||
write("cache", `{"module":"cache","resources":[{"id":"server","type":"container","image":"cache@sha256:a","health":{"kind":"tcp","endpoint":"redis"}},
|
||||
{"id":"plain","type":"container","image":"cache@sha256:a"}]}`)
|
||||
show := func(ref, path string) ([]byte, error) {
|
||||
return exec.Command("git", "-C", root, "show", ref+":"+path).Output()
|
||||
}
|
||||
changed, err := ChangedHealth(root, "main", show)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range changed {
|
||||
got = append(got, r.Module+"."+r.ID)
|
||||
}
|
||||
if strings.Join(got, ",") != "cache.server,web.server" {
|
||||
t.Fatalf("the bed would prove %v; want the new module's checked container and the one whose image moved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func orDefault(s, fallback string) string {
|
||||
if s == "" {
|
||||
return fallback
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package replays
|
||||
|
||||
import (
|
||||
|
||||
+181
-8
@@ -7,7 +7,9 @@
|
||||
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
|
||||
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
|
||||
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
|
||||
// resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's.
|
||||
// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its
|
||||
// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed
|
||||
// through to the controller's.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -20,6 +22,8 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -42,8 +46,8 @@ func main() {
|
||||
if r.Before != "" {
|
||||
at = r.Before
|
||||
}
|
||||
before, beforeSaid := run(r, *repos, at)
|
||||
after, afterSaid := run(r, *repos, r.Fix)
|
||||
before, beforeSaid := run(r, *repos, at, false)
|
||||
after, afterSaid := run(r, *repos, r.Fix, true)
|
||||
verdict := "PROVED"
|
||||
if before != "fails" && before != "absent" || after != "passes" {
|
||||
verdict = "NOT PROVED"
|
||||
@@ -62,8 +66,13 @@ func main() {
|
||||
}
|
||||
|
||||
// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
|
||||
func run(r replays.Replay, repos, at string) (string, string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
|
||||
limit := 15 * time.Minute
|
||||
if r.Kind == replays.Bed {
|
||||
// Two machines raised, a foundation pulled and a control plane built inside one of them.
|
||||
limit = 90 * time.Minute
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), limit)
|
||||
defer cancel()
|
||||
repo := filepath.Join(repos, r.Repository)
|
||||
tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
|
||||
@@ -77,7 +86,9 @@ func run(r replays.Replay, repos, at string) (string, string) {
|
||||
defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
|
||||
|
||||
switch r.Kind {
|
||||
case replays.InRepository, replays.Gate:
|
||||
case replays.Bed:
|
||||
return aBed(ctx, r, repos, tree, fixed)
|
||||
case replays.InRepository, replays.Gate, replays.Liveness:
|
||||
home := filepath.Join(repos, r.Repository)
|
||||
if r.Home != "" {
|
||||
home = filepath.Join(repos, r.Home)
|
||||
@@ -95,12 +106,44 @@ func run(r replays.Replay, repos, at string) (string, string) {
|
||||
return "error", err.Error()
|
||||
}
|
||||
}
|
||||
if r.Kind == replays.Liveness {
|
||||
// The other repositories at their own commits on the same side, and the replay here run
|
||||
// against all of them.
|
||||
env := []string{"MESH_REPLAY_CONTROLLER=" + tree}
|
||||
for _, also := range r.With {
|
||||
ref := also.Fix
|
||||
if !fixed {
|
||||
ref = also.Fix + "^1"
|
||||
if also.Before != "" {
|
||||
ref = also.Before
|
||||
}
|
||||
}
|
||||
other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
defer os.RemoveAll(other)
|
||||
clone := filepath.Join(repos, also.Repository)
|
||||
if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
|
||||
ref).CombinedOutput(); err != nil {
|
||||
return "error", string(out)
|
||||
}
|
||||
defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
|
||||
if also.Repository == "mesh-host" {
|
||||
env = append(env, "MESH_REPLAY_HOST="+other)
|
||||
}
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package)
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return verdictOf(string(out), err, false)
|
||||
}
|
||||
mode := "-mod=vendor"
|
||||
if _, err := os.Stat(filepath.Join(tree, "vendor")); err != nil {
|
||||
if _, err := os.Stat(filepath.Join(tree, r.Module, "vendor")); err != nil {
|
||||
mode = "-mod=mod"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
|
||||
cmd.Dir = tree
|
||||
cmd.Dir = filepath.Join(tree, r.Module)
|
||||
cmd.Env = append(os.Environ(), "GOFLAGS="+mode, "GOPRIVATE=git.novox.be")
|
||||
out, err := cmd.CombinedOutput()
|
||||
return verdictOf(string(out), err, r.Kind == replays.Gate)
|
||||
@@ -123,6 +166,136 @@ func run(r replays.Replay, repos, at string) (string, string) {
|
||||
return "error", "no such kind"
|
||||
}
|
||||
|
||||
// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its
|
||||
// installer built from it, the installer carrying a builder built from the controller the replay pins, and
|
||||
// the bed run with them, the pinned controller to build from and the pinned catalogue to read.
|
||||
func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) {
|
||||
beside := map[string]string{}
|
||||
for _, also := range r.With {
|
||||
ref := also.Fix
|
||||
if !fixed {
|
||||
ref = also.Fix + "^1"
|
||||
if also.Before != "" {
|
||||
ref = also.Before
|
||||
}
|
||||
}
|
||||
other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
defer os.RemoveAll(other)
|
||||
clone := filepath.Join(repos, also.Repository)
|
||||
if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
|
||||
ref).CombinedOutput(); err != nil {
|
||||
return "error", string(out)
|
||||
}
|
||||
defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
|
||||
beside[also.Repository] = other
|
||||
}
|
||||
controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"]
|
||||
if controller == "" || catalogue == "" {
|
||||
return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " +
|
||||
"not both"
|
||||
}
|
||||
builder, err := builderImage(ctx, controller)
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap")
|
||||
for _, step := range [][]string{
|
||||
{"make", "build", "SYSTEM=arch"},
|
||||
{"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer},
|
||||
} {
|
||||
cmd := exec.CommandContext(ctx, step[0], step[1:]...)
|
||||
cmd.Dir = tree
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6))
|
||||
}
|
||||
}
|
||||
lab, err := filepath.Abs("..")
|
||||
if err != nil {
|
||||
return "error", err.Error()
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test)
|
||||
cmd.Dir = lab
|
||||
cmd.Env = append(os.Environ(),
|
||||
"MESH_LAB_HOST_BINARY="+host,
|
||||
"MESH_LAB_BOOTSTRAP_BINARY="+installer,
|
||||
// The template the installer knows how to make safe: it rotates the store's and the broker's
|
||||
// well-known credentials, and refuses a template naming neither (the bus one names no broker).
|
||||
"MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"),
|
||||
"MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"),
|
||||
"MESH_REPLAY_CONTROLLER="+controller)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return bedVerdict(string(out), err)
|
||||
}
|
||||
|
||||
// builderImage is the builder the installer carries, built from the controller at the commit the replay
|
||||
// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit.
|
||||
func builderImage(ctx context.Context, controller string) (string, error) {
|
||||
head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tag := "mesh-builder:prove-" + strings.TrimSpace(string(head))
|
||||
if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
|
||||
return tag, nil
|
||||
}
|
||||
mod, err := os.ReadFile(filepath.Join(controller, "go.mod"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
release := ""
|
||||
for _, line := range strings.Split(string(mod), "\n") {
|
||||
if v, ok := strings.CutPrefix(line, "go "); ok {
|
||||
parts := strings.Split(strings.TrimSpace(v), ".")
|
||||
if len(parts) >= 2 {
|
||||
release = parts[0] + "." + parts[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
if release == "" {
|
||||
return "", errors.New("the controller's go.mod names no Go release to build its builder on")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine",
|
||||
"-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".")
|
||||
cmd.Dir = controller
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6))
|
||||
}
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was
|
||||
// not given what it raises — and never the fault reproduced.
|
||||
func bedVerdict(out string, err error) (string, string) {
|
||||
count := func(what string) int {
|
||||
m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return -1
|
||||
}
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
return n
|
||||
}
|
||||
pass, fail, skipped := count("pass"), count("fail"), count("skipped")
|
||||
switch {
|
||||
case skipped > 0:
|
||||
said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out)
|
||||
if said != nil {
|
||||
return "error", "the bed skipped: " + said[1]
|
||||
}
|
||||
return "error", "the bed skipped: " + lastLines(out, 6)
|
||||
case fail > 0:
|
||||
if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil {
|
||||
return "fails", strings.TrimSpace(m[1])
|
||||
}
|
||||
return "fails", lastLines(out, 6)
|
||||
case pass > 0 && err == nil:
|
||||
return "passes", lastLines(out, 3)
|
||||
}
|
||||
return "error", lastLines(out, 6)
|
||||
}
|
||||
|
||||
func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
|
||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
|
||||
cmd.Env = append(os.Environ(), env)
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
//go:build integration
|
||||
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **R-crashloop — a module's container that crash-loops fails its gate on the first machine** (novox/hq
|
||||
// ADR 0240, "how it is checked", rule 1; to-be 48 Phase A, done when).
|
||||
//
|
||||
// On the home server the agent server restarted about a hundred times in a crash loop while the mesh read
|
||||
// it applied, its tools served and no condition raised; a person found it reading its log for another
|
||||
// reason (issue 268, research 032 §6). The gate judged a module by what the mesh saw from outside.
|
||||
//
|
||||
// The replay is the whole chain, each half at its commit:
|
||||
//
|
||||
// 1. a container whose program exits at start is raised here as the node-engine raises a module's — the
|
||||
// runtime restarting it unless stopped, labelled with its resource;
|
||||
// 2. the node-engine at MESH_REPLAY_HOST judges it through the runtime, as it judges every long-running
|
||||
// resource, and writes what it states (its TestReplayCrashLoopIsSaidUnhealthy) — an engine older than
|
||||
// the judging has nothing to state, and its reports carry no health;
|
||||
// 3. the controller at MESH_REPLAY_CONTROLLER rolls a build of the module out, hears its first machine say
|
||||
// that, and judges the gate (its TestReplayCrashLoopFailsItsGateOnTheFirstMachine).
|
||||
//
|
||||
// The assertion is the controller's: the build fails its gate on the first machine and is put back there.
|
||||
// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is
|
||||
// removed after, whatever happened.
|
||||
func TestReplayCrashLoop(t *testing.T) {
|
||||
host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host"))
|
||||
controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller"))
|
||||
for _, dir := range []string{host, controller} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir)
|
||||
}
|
||||
}
|
||||
if os.Getenv("MESH_TEST_POSTGRES") == "" {
|
||||
t.Skip("no MESH_TEST_POSTGRES: the controller's half judges its gate against a store")
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the crash loop is a container")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 1. The crash loop, as the node-engine raises a module's container.
|
||||
const image = "alpine:3.20"
|
||||
if err := docker.Pull(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
|
||||
id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"sh", "-c", "echo starting; exit 3"}, Name: name,
|
||||
Restart: "unless-stopped", Labels: map[string]string{"mesh-host.id": "app.server"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), id)
|
||||
|
||||
// 2. The node-engine at its commit judges it, and says what it states.
|
||||
statement := filepath.Join(t.TempDir(), "statement.json")
|
||||
engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopIsSaidUnhealthy$",
|
||||
"./internal/liveness/")
|
||||
engine.Dir = host
|
||||
engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=")
|
||||
out, err := engine.CombinedOutput()
|
||||
switch {
|
||||
case err == nil && !strings.Contains(string(out), "no tests to run"):
|
||||
t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement)))
|
||||
case judgesNothing(string(out)):
|
||||
// An engine older than the judging: its reports carry no health.
|
||||
t.Logf("the node-engine at this commit judges nothing it runs")
|
||||
if err := os.WriteFile(statement, []byte("null"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("the node-engine did not say the crash loop unhealthy: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
|
||||
// 3. The controller at its commit judges the gate from what the engine said.
|
||||
gate := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopFailsItsGateOnTheFirstMachine$",
|
||||
"./cmd/mesh-controller/")
|
||||
gate.Dir = controller
|
||||
gate.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off")
|
||||
out, err = gate.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("the gate did not fail the crash loop on its first machine: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
if strings.Contains(string(out), "no tests to run") {
|
||||
t.Fatal("the controller at this commit has no half of the replay to run")
|
||||
}
|
||||
}
|
||||
|
||||
// judgesNothing is go test's word for a package that is not there: an engine older than the judging.
|
||||
func judgesNothing(out string) bool {
|
||||
return strings.Contains(out, "no tests to run") || strings.Contains(out, "directory not found") ||
|
||||
strings.Contains(out, "no Go files") || strings.Contains(out, "cannot find package") ||
|
||||
strings.Contains(out, "matched no packages") || strings.Contains(out, "is not in std")
|
||||
}
|
||||
|
||||
func statementOf(path string) string {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "(nothing written: " + err.Error() + ")"
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func firstLineOf(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
if len(line) > 400 {
|
||||
line = line[:400] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func lastOf(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
+72
-6
@@ -1,6 +1,7 @@
|
||||
package replays
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
@@ -74,11 +75,21 @@ func (d *Docker) Pull(ctx context.Context, image string) error {
|
||||
if err := d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil); err == nil {
|
||||
return nil
|
||||
}
|
||||
name, tag, _ := strings.Cut(image, ":")
|
||||
if tag == "" {
|
||||
tag = "latest"
|
||||
// A digest is the tag the runtime is asked for, as its own client asks: `name@sha256:…` cut at the
|
||||
// `@`; otherwise the tag after the last `:` that is not part of a registry's port.
|
||||
name, tag, digested := strings.Cut(image, "@")
|
||||
if !digested {
|
||||
name, tag = image, "latest"
|
||||
if at := strings.LastIndex(image, ":"); at > strings.LastIndex(image, "/") {
|
||||
name, tag = image[:at], image[at+1:]
|
||||
}
|
||||
}
|
||||
return d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil)
|
||||
err := d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The runtime answers a pull it could not make with a stream that ends in an error, not a status.
|
||||
return d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil)
|
||||
}
|
||||
|
||||
// Network makes a network of its own and answers its id.
|
||||
@@ -98,9 +109,15 @@ func (d *Docker) RemoveNetwork(ctx context.Context, id string) {
|
||||
type Run struct {
|
||||
Image string
|
||||
Cmd []string
|
||||
Env []string
|
||||
Binds []string
|
||||
Network string
|
||||
DNS []string
|
||||
// Name, Labels and Restart are the container's name, its labels beside the replay's own, and the
|
||||
// runtime's restart policy — what the node-engine creates a module's container with.
|
||||
Name string
|
||||
Labels map[string]string
|
||||
Restart string
|
||||
}
|
||||
|
||||
// Start creates and starts a container and answers its id and its address on its network.
|
||||
@@ -110,8 +127,19 @@ func (d *Docker) Start(ctx context.Context, r Run) (string, string, error) {
|
||||
if len(r.DNS) > 0 {
|
||||
host["Dns"] = r.DNS
|
||||
}
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/create", map[string]any{"Image": r.Image, "Cmd": r.Cmd,
|
||||
"Labels": map[string]string{d.Label: "1"}, "HostConfig": host}, &made); err != nil {
|
||||
if r.Restart != "" {
|
||||
host["RestartPolicy"] = map[string]any{"Name": r.Restart}
|
||||
}
|
||||
labels := map[string]string{d.Label: "1"}
|
||||
for k, v := range r.Labels {
|
||||
labels[k] = v
|
||||
}
|
||||
path := "/containers/create"
|
||||
if r.Name != "" {
|
||||
path += "?name=" + url.QueryEscape(r.Name)
|
||||
}
|
||||
if err := d.call(ctx, http.MethodPost, path, map[string]any{"Image": r.Image, "Cmd": r.Cmd,
|
||||
"Env": r.Env, "Labels": labels, "HostConfig": host}, &made); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {
|
||||
@@ -194,3 +222,41 @@ func (d *Docker) Exec(ctx context.Context, id string, cmd ...string) (int, error
|
||||
}
|
||||
return -1, fmt.Errorf("%v did not end", cmd)
|
||||
}
|
||||
|
||||
// Build builds an image from a Dockerfile alone, tagged as given, and labelled so a replay that dies is
|
||||
// cleaned up by it.
|
||||
func (d *Docker) Build(ctx context.Context, tag, dockerfile string) error {
|
||||
var archive bytes.Buffer
|
||||
tw := tar.NewWriter(&archive)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "Dockerfile", Mode: 0o644, Size: int64(len(dockerfile))}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write([]byte(dockerfile)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
labels, _ := json.Marshal(map[string]string{d.Label: "1"})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://docker/build?t="+url.QueryEscape(tag)+
|
||||
"&labels="+url.QueryEscape(string(labels))+"&rm=1", &archive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-tar")
|
||||
res, err := d.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
said, _ := io.ReadAll(res.Body)
|
||||
if res.StatusCode >= 300 || bytes.Contains(said, []byte(`"error"`)) {
|
||||
return fmt.Errorf("building %s: %s %s", tag, res.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveImage removes an image.
|
||||
func (d *Docker) RemoveImage(ctx context.Context, ref string) {
|
||||
_ = d.call(ctx, http.MethodDelete, "/images/"+url.PathEscape(ref)+"?force=1", nil, nil)
|
||||
}
|
||||
|
||||
@@ -28,8 +28,27 @@ const (
|
||||
// Gate is a test of the merge gate itself: before its fix there was no check to fail, so the commit
|
||||
// before is held to fail by not having it.
|
||||
Gate Kind = "gate"
|
||||
// Liveness is the crash-loop replay here: a container that exits at start, raised on the runtime here,
|
||||
// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
|
||||
// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
|
||||
Liveness Kind = "liveness"
|
||||
// Bed is a bed of the lab, raised on its hypervisor: Test is its file under test/integration, run with
|
||||
// the node-engine and its installer built from Repository at the commit, beside the With repositories at
|
||||
// theirs (novox/hq issue 92). A bed that could not reach the step it replays skips, and the prover reads
|
||||
// a skip as an error, never as the fault reproduced.
|
||||
Bed Kind = "bed"
|
||||
)
|
||||
|
||||
// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
|
||||
// it, as Repository is.
|
||||
type Also struct {
|
||||
Repository string
|
||||
Fix string
|
||||
// Before is the commit the replay runs this repository at before the fix, when that is not Fix's
|
||||
// first parent.
|
||||
Before string
|
||||
}
|
||||
|
||||
// Replay is one incident, replayed.
|
||||
type Replay struct {
|
||||
ID string
|
||||
@@ -51,6 +70,11 @@ type Replay struct {
|
||||
Files []string
|
||||
// Home is where the replay's files live, when that is not the fix: the repository and ref.
|
||||
Home, HomeRef string
|
||||
// With is the other repositories the replay runs at their commits beside Repository's (Liveness).
|
||||
With []Also
|
||||
// Module is the directory of the Go module the replay runs in, within Repository, when that is not its
|
||||
// root: a catalogue module is a Go module of its own. Package is relative to it; Files are not.
|
||||
Module string
|
||||
}
|
||||
|
||||
// Register is every replay, by incident.
|
||||
@@ -78,6 +102,553 @@ var Register = []Replay{
|
||||
Asserts: "a consumer beside its store stays bound to it while another machine holds the store's seat",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay273", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "feat/replays"},
|
||||
// The controller's incidents of 2026-10-07, each a test laid over the merge before its fix. Home is
|
||||
// the controller's commit that added them; it stays reachable once that pull request merges.
|
||||
{ID: "R296", Issue: 296, Kind: InRepository, Repository: "mesh-controller", Fix: "3808634",
|
||||
What: "a plan's clock restarted at every save: a build queued for minutes read \"building for 1s\", " +
|
||||
"then 4s, then 9s, and was never LATE",
|
||||
Asserts: "a plan's line counts from when it entered its tier, and a plan standing still in its tier is not " +
|
||||
"saved again",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay296", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
{ID: "R299", Issue: 299, Kind: InRepository, Repository: "mesh-controller", Fix: "c37d774",
|
||||
What: "a seat's first verbs, all optional, made the self-check say its holder silent on every machine",
|
||||
Asserts: "a holder of a seat whose verbs are all optional is not said silent when it answers nothing",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay299", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
{ID: "R300", Issue: 300, Kind: InRepository, Repository: "mesh-controller", Fix: "e7dca6c",
|
||||
What: "a merge that added a module said it changed nothing the mesh holds, and built nothing, so the " +
|
||||
"module was never registered",
|
||||
Asserts: "a merge adding a module asks the build seat for it, at the branch merged into, and opens no plan",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay300", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
// A drill counted as a repair (2026-10-07): S15 read two deliberate drills, recorded through hand-act
|
||||
// record for want of a verb, as a cause repaired twice. In a file of its own, since replays_test.go holds
|
||||
// replays of later commits that do not build at the commit before this fix. Home is the controller's
|
||||
// commit that added it; it stays reachable once that pull request merges.
|
||||
{ID: "R292", Issue: 292, Kind: InRepository, Repository: "mesh-controller", Fix: "863ebd4",
|
||||
What: "two drills of ADR 0240, each with the operator's word, recorded through hand-act record --cause drill, " +
|
||||
"raised mesh.hand-acts.drill.healer-wanted",
|
||||
Asserts: "a drill recorded in the hand-act log wants no healer, however it was recorded",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay292", Files: []string{"cmd/mesh-controller/replay292_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
||||
// A seat's new verb deadlocked across two repositories (2026-10-07): the controller promising checks
|
||||
// refused the delivery seat's holder that did not serve it, and a holder serving it was refused by the
|
||||
// controller that did not promise it.
|
||||
{ID: "R298", Issue: 298, Kind: InRepository, Repository: "mesh-controller", Fix: "f6aea4b",
|
||||
What: "the delivery seat's new verb checks could land in neither repository first: each side's claim " +
|
||||
"check refused the other's holder",
|
||||
Asserts: "the delivery seat's holder holds it both before and after it serves checks",
|
||||
Package: "./internal/catalogue", Test: "TestReplay298", Files: []string{"internal/catalogue/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
||||
// The push a recorded build waits for, counted as a repair (2026-10-07): a test in the controller, in
|
||||
// the fix's own commit, recording two such pushes as the seat's push records them.
|
||||
{ID: "R301", Issue: 301, Kind: InRepository, Repository: "mesh-controller", Fix: "e92a3fe",
|
||||
What: "the operator's pushes of new builds whose upgrade policy is record — the resolver, the network " +
|
||||
"managers, the packet filter — were counted as repairs, and S15 wanted a healer for each cause",
|
||||
Asserts: "a push that moves only recorded builds, recorded as the push records it, wants no healer " +
|
||||
"however often its cause is given",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay301", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
||||
// The lab's Go replays never compiled before a lab merge (2026-10-07): the build agent ran a repository's
|
||||
// check in one toolchain. Before the fix there was no part to run, so the commit before is held to fail
|
||||
// by not having it.
|
||||
{ID: "R302", Issue: 302, Kind: Gate, Repository: "mesh-controller", Fix: "98ef7ba",
|
||||
What: "mesh-lab's replays register merged with NOT CHECKED HERE: its check ran in the TypeScript " +
|
||||
"toolchain, which holds no Go compiler",
|
||||
Asserts: "a repository in two languages has each declared part of its own check run in that part's " +
|
||||
"toolchain, and the layer passes only when every part does",
|
||||
Package: "./internal/builder", Test: "TestARepositoryInTwoLanguagesIsCheckedInBoth",
|
||||
Files: []string{"internal/builder/check_test.go"}},
|
||||
// A recheck left the old verdict standing (2026-10-07): the forge keeps the newest status of each context
|
||||
// on a head, and a recheck asks of the same head, so the old green let the pull request merge while the
|
||||
// fresh check ran. A test in mesh-delivery, which asks the forge's holder (core) to put the merge check
|
||||
// back to pending; the holder's own half is its TypeScript test.
|
||||
{ID: "R305", Issue: 305, Kind: InRepository, Repository: "mesh-catalog", Fix: "74f912e",
|
||||
What: "a pull request rechecked at 23:04 kept mesh/merge-gate and mesh/repo-check success from 16:59, " +
|
||||
"and the forge would have merged it while its fresh check ran",
|
||||
Asserts: "a recheck puts the head's merge check statuses back to pending before it asks the check, and " +
|
||||
"is refused whole when the forge cannot be told",
|
||||
Module: "modules/mesh-delivery", Package: "./cmd/mesh-delivery", Test: "TestARecheck",
|
||||
Files: []string{"modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go",
|
||||
"modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go"}},
|
||||
// A pull request ran its own copy of the repository's check (2026-10-08): the build agent read
|
||||
// merge-check.sh from the change's head, so a branch cut before the script, or one that deleted or
|
||||
// gutted it, merged with none of its tests run. Written with only what the agent had before the fix, so
|
||||
// it fails on the commit before by answering the warning. The fix is still on its branch: proved
|
||||
// against the main it branched from.
|
||||
{ID: "R310", Issue: 310, Kind: InRepository, Repository: "mesh-controller", Fix: "cb0327d",
|
||||
Before: "1a50d6e",
|
||||
What: "the media catalogue's Lidarr pull request, cut before its repository had a merge-check.sh, was " +
|
||||
"answered no repository check defined as a warning and merged with none of its tests run",
|
||||
Asserts: "where the base branch holds a merge-check.sh, its script judges the change's tree, and a head " +
|
||||
"without one is never answered the warning",
|
||||
Package: "./internal/builder", Test: "TestReplay310", Files: []string{"internal/builder/replay310_test.go"}},
|
||||
// Two order rules ordered one pair both ways (2026-10-08): the controller's member moved the build agent,
|
||||
// which builds the node-engine, and the node-engine goes before the controller, so the group
|
||||
// feat/a-machine-joins-through-the-tunnel was refused as a cycle and merged by hand. A test in the
|
||||
// controller, using only what orderOf had before its fix, laid over the commit before it.
|
||||
{ID: "R309", Issue: 309, Kind: InRepository, Repository: "mesh-controller", Fix: "c5a2edf",
|
||||
What: "a delivery group of the controller, the node-engine and the lab, each ready, was rejected: built by " +
|
||||
"put the controller's member first and engine before controller put the node-engine's first",
|
||||
Asserts: "rules that order one pair both ways are resolved by precedence — built by over engine before " +
|
||||
"controller — and the group is ordered, the controller's member first",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay309", Files: []string{"cmd/mesh-controller/replay309_test.go"}},
|
||||
// An answer larger than one message of the bus (2026-10-08): the controller's conditions and status
|
||||
// outgrew the bus's max_payload once 689 stalled deliveries were open conditions, the client library
|
||||
// refused every reply inside the controller, and every caller timed out while calls said answered.
|
||||
{ID: "R314", Issue: 314, Kind: InRepository, Repository: "mesh-controller", Fix: "175b28e",
|
||||
What: "conditions and status answered three times what one message of the bus carries; the reply was " +
|
||||
"refused inside the controller and every caller waited 30 s and read that it did not answer",
|
||||
Asserts: "an answer larger than one message arrives whole to a caller that pages, and one that does not " +
|
||||
"is told so in a message that fits — never a timeout",
|
||||
Package: "./internal/link", Test: "TestReplay314", Files: []string{"internal/link/replay314_test.go"}},
|
||||
// One module's wait for a new login held every other module's walk (2026-10-08): the walk of the builds
|
||||
// waiting for a gate sent the laptop three moves in one send, the lighting module's account said relogin
|
||||
// needed and its daemon's unit failed in that account's manager, and the gate failed the send at its bound,
|
||||
// put the module back (taking its account group back) and stopped every walk behind it. A test in the
|
||||
// controller, reading the statements as the node-engine's JSON, so it is laid over the commit before.
|
||||
// Fix is the head of the fix: e3b5c22 alone raises the new condition kind without the plain words the
|
||||
// controller's suite requires since hq ADR 0253, which 51d7bbf gives it. Before is the main both stood on.
|
||||
{ID: "R318", Issue: 318, Kind: InRepository, Repository: "mesh-controller", Fix: "51d7bbf", Before: "6df30b6",
|
||||
What: "a send of three modules to the laptop failed its gate after ten minutes because one module's account " +
|
||||
"needed a new login; the module was put back and the walks of the other modules stopped behind it",
|
||||
Asserts: "a wait for a person's new login passes the gate with the wait carried and said as the module's " +
|
||||
"relogin-needed condition; nothing is put back, and the walk goes on to the next machine",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay318", Files: []string{"cmd/mesh-controller/replay318_test.go"}},
|
||||
// An assignment answered as unknown while its module's build ran (2026-10-08): the merge adding `sensors`
|
||||
// asked for its build (issue 300), and `assign g14 sensors` a minute later said "no module of that name",
|
||||
// which read as a module nobody registered. A test in the controller, written with only what it had
|
||||
// before the fix, laid over the commit before. The fix is still on its branch: Fix is its pull request's
|
||||
// head, proved against the main it is rebased on.
|
||||
{ID: "R325", Issue: 325, Kind: InRepository, Repository: "mesh-controller", Fix: "8adb7f1", Before: "fe00fec",
|
||||
What: "a minute after the merge that added sensors asked for its build, assign g14 sensors answered no " +
|
||||
"module of that name, and the same call worked minutes later",
|
||||
Asserts: "an assignment made while its module's build runs says the build, is kept, and is made when the " +
|
||||
"build registers the module",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay325", Files: []string{"cmd/mesh-controller/replay325_test.go"}},
|
||||
// A declared tool check never asked on a live machine (2026-10-08): the queue reached the bus through an
|
||||
// assertion only OverNATS met, and the NATS link the node-engine attaches had no Ask, so every tool check
|
||||
// read "no link to the bus is open" while the link was up. A test in the node-engine, written with only
|
||||
// what it had before the fix; Home is its commit that added it.
|
||||
{ID: "R331", Issue: 331, Kind: InRepository, Repository: "mesh-host", Fix: "824911d",
|
||||
What: "the sensors module's tool check on the laptop read no link to the bus is open for hours while the " +
|
||||
"node-engine's link was up and the same tool answered through the mesh MCP server",
|
||||
Asserts: "a tool check on the NATS link the node-engine attaches reaches that link, and is not refused as no link",
|
||||
Package: "./internal/link", Test: "TestReplay331", Files: []string{"internal/link/replay331_test.go"},
|
||||
Home: "mesh-host", HomeRef: "5684a4579d92"},
|
||||
// Messages a consumer gave up on were kept nowhere (2026-10-06): a media server's consumer gave up on
|
||||
// several downloads after five deliveries each, the controller's condition cleared within minutes, and
|
||||
// EVENTS dropped them a week later; the dead-letter stream design 25 promised did not exist. A test in
|
||||
// the controller, written with only what the link had before the fix, laid over the commit before. Fix is
|
||||
// the merge of its pull request, proved against its first parent.
|
||||
{ID: "R330", Issue: 330, Kind: InRepository, Repository: "mesh-controller", Fix: "ca09a07",
|
||||
What: "a media server's event consumer gave up on several messages after five deliveries each; the condition " +
|
||||
"cleared within minutes and nothing kept which messages they were",
|
||||
Asserts: "a consumer that never acknowledges an event gives it up, and the event is kept on the bus with " +
|
||||
"its consumer, stream, subject, deliveries and when it was given up",
|
||||
Package: "./internal/link", Test: "TestReplay330", Files: []string{"internal/link/replay330_test.go"}},
|
||||
// Every verb opened a bus connection of its own (2026-10-08): the server's connection total rose by 5.2 a
|
||||
// minute while the same 16 stayed open, and three conditions calls in one second added three, each a
|
||||
// process of the controller's binary that dialled, logged in and left. A test in the controller, written
|
||||
// with only what it had before the fix, laid over the commit before. Fix is the merge of its pull
|
||||
// request, proved against its first parent.
|
||||
{ID: "R327", Issue: 327, Kind: InRepository, Repository: "mesh-controller", Fix: "efcdd5d",
|
||||
What: "three calls of conditions in one second added three connections to the bus: each verb ran as a " +
|
||||
"process that dialled it, and hundreds of such connections an hour hid any client reconnecting",
|
||||
Asserts: "a verb that only reads, served by the serving controller, opens no connection of its own",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay327", Files: []string{"cmd/mesh-controller/replay327_test.go"}},
|
||||
// A passed build put back half an hour after its pass (2026-10-08): dunst passed its gate on the laptop,
|
||||
// its send to the rest waited on another walk, that walk sent the laptop a declaration it did not report
|
||||
// on, and the plan read the silence as the passed build never applied: failed at its gate with the pass's
|
||||
// own words, and put back. A test in the controller, written with only what it had before the fix, laid
|
||||
// over the commit before. Fix is the merge of the fix.
|
||||
{ID: "R335", Issue: 335, Kind: InRepository, Repository: "mesh-controller", Fix: "8170fc5",
|
||||
What: "a build that passed its gate on the laptop (healthy 3 times over 2m5s) was marked failed at its " +
|
||||
"gate 28 minutes later, with those words as the reason, and put back",
|
||||
Asserts: "once a build passed its gate, the first machine's later reports neither fail it nor hold the rest, " +
|
||||
"and a walk stopped after the pass keeps it",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay335", Files: []string{"cmd/mesh-controller/replay335_test.go"}},
|
||||
// A send held for the bus's planned step said nothing (2026-10-08): a catalogue merge built a new bus for the
|
||||
// control-node, every send to that machine was refused for 28 minutes for a step only a person starts, and
|
||||
// nothing but the walks' notes said so. A test in the controller, written with only what it had before the
|
||||
// fix, laid over the commit before. Fix is the merge of its pull request (mesh-controller #173), proved
|
||||
// against its first parent.
|
||||
{ID: "R336", Issue: 336, Kind: InRepository, Repository: "mesh-controller", Fix: "b13a0f7",
|
||||
What: "for 28 minutes every send to the control-node was refused for a new bus build waiting for a " +
|
||||
"person's bus upgrade, and no condition said so",
|
||||
Asserts: "the first watchdog tick after a send is refused for the bus's planned step opens a condition for " +
|
||||
"the operator naming what waits and the call that ends it, and it clears once the machine is sent the " +
|
||||
"new bus",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay336", Files: []string{"cmd/mesh-controller/replay336_test.go"}},
|
||||
// Two merges seconds apart left their delivery waiting for a person (2026-10-10): two catalogue merges 18 s
|
||||
// apart were walked by one walk at the later commit, which superseded the first; mesh-delivery tied it to the
|
||||
// delivery the supersession had ended, so nothing gave it its word for 58 minutes. A test in mesh-delivery,
|
||||
// written with only what it had before the fix, as the controller of that night said its walks. The fix is
|
||||
// still on its branch (mesh-catalog #171): proved against the main it branched from.
|
||||
{ID: "R362", Issue: 362, Kind: InRepository, Repository: "mesh-catalog", Fix: "13ffae7",
|
||||
Before: "52f34d3",
|
||||
What: "claude-code and dunst merged 18 s apart; the one walk at the dunst commit was tied to the dunst " +
|
||||
"delivery its supersession had ended, never let go, and claude-code was held for a person after ten minutes",
|
||||
Asserts: "the walk at the later commit is let go once and delivers both merges without a person — dunst " +
|
||||
"delivered, claude-code covered by it — whichever of the controller's two records arrives first",
|
||||
Module: "modules/mesh-delivery", Package: "./cmd/mesh-delivery", Test: "TestReplay362",
|
||||
Files: []string{"modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go",
|
||||
"modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go"}},
|
||||
// A path change read as data loss (2026-10-10): the agent's home on the control-node moved from the
|
||||
// operator's own home to the agent account's (ADR 0266), and data-shrank was raised because the fresh
|
||||
// directory was read against the old one's size. A test in the controller, written with only what it had
|
||||
// before the fix, the incident replayed through the store into D13's findings. Fix is the merge of its
|
||||
// pull request (mesh-controller #204), proved against its first parent.
|
||||
{ID: "R368", Issue: 368, Kind: InRepository, Repository: "mesh-controller", Fix: "798738c",
|
||||
What: "agent-home of claude-code on the control-node was raised as shrunk to 490 B from 94.7 MB when its " +
|
||||
"path moved to the agent account's fresh home; nothing was lost",
|
||||
Asserts: "an item whose path moved raises no data-shrank, and a genuine shrink at its new path is still " +
|
||||
"raised, urgent for what is irreplaceable",
|
||||
Package: "./cmd/mesh-controller", Test: "TestAMovedPathIsNoShrinkAndAShrinkThereStillIs",
|
||||
Files: []string{"cmd/mesh-controller/data_test.go"}},
|
||||
// A quote that could not be said (2026-10-07): a machine's role, a JSON value in single quotes with an
|
||||
// apostrophe inside, was cut at the apostrophe by the controller's `command` verb, and the line was refused
|
||||
// as a bare unclosed quote, naming neither where nor how a quote is written. A test in the controller,
|
||||
// written with only what it had before the fix. Fix is the merge of its pull request (mesh-controller
|
||||
// #205), proved against its first parent.
|
||||
{ID: "R294", Issue: 294, Kind: InRepository, Repository: "mesh-controller", Fix: "690b75f",
|
||||
What: "settings set claude-code '{\"role\":\"the operator's laptop\"}' through `command` was cut at the " +
|
||||
"apostrophe and refused as an unclosed quote, with no position and no way to write a quote",
|
||||
Asserts: "a quote in a value reads back through both shell forms, and a line left with an open quote is " +
|
||||
"refused naming the character and how a quote is written, quoting none of the line",
|
||||
Package: "./cmd/mesh-controller", Test: "TestAQuotedValueCanHoldAQuote",
|
||||
Files: []string{"cmd/mesh-controller/seatverbs_test.go"}},
|
||||
// A mesh directory outside every home was changed by path (2026-10-12): the directory applier set a
|
||||
// directory's mode and owner by calling chmod and chown on its path, which follows a link anywhere in
|
||||
// the path, so an account able to swap a parent for a link between two applies could direct root's
|
||||
// chmod and chown at the link's target. Found by code reading, no known exploit. A test in the
|
||||
// node-engine, written only with what applyDirectory had before its fix, laid over the commit before
|
||||
// (mesh-host #87). Fix is the merge of its pull request.
|
||||
{ID: "R358", Issue: 358, Kind: InRepository, Repository: "mesh-host", Fix: "36704b5471cb7b4e585847a08e4fd7fd64aea357",
|
||||
What: "a mesh directory outside every home was given its mode and owner by path, so a parent swapped " +
|
||||
"for a link between two applies carried root's chmod and chown to the link's target",
|
||||
Asserts: "no apply changes the mode or owner of anything but the directory its declaration names, " +
|
||||
"however a parent of its path is swapped for a link between applies, and the refusal names the link " +
|
||||
"and the account that could have made it",
|
||||
Package: "./internal/apply", Test: "TestAParentSwappedForALinkLeavesItsTargetUntouched",
|
||||
Files: []string{"internal/apply/replay358_test.go"}},
|
||||
// assign said two things that could not both be true (2026-10-04): a workstation already ran a module
|
||||
// contributing node-hotkeys, and the module holding that seat was assigned to it; the answer said in
|
||||
// order that the seat was now held, then that the node was left out of the rest of the mesh for lack
|
||||
// of it, naming the assignment just made as the remedy. `plan` for the node right after resolved. A
|
||||
// test in the controller, written with only what it had before its fix — register, assign's command,
|
||||
// planFor — laid over the commit before. Needs a store: raised by inventory.ForTest against
|
||||
// MESH_TEST_POSTGRES, same as the controller's other store-backed replays. Fix is the merge of its
|
||||
// pull request (mesh-controller #253).
|
||||
{ID: "R237", Issue: 237, Kind: InRepository, Repository: "mesh-controller", Fix: "5facd8b08e245545fb7656dd4af11280d9306a08",
|
||||
What: "assign on a workstation already running a module that contributed node-hotkeys, after " +
|
||||
"assigning the module holding that seat, said the seat was now held and then that the node was " +
|
||||
"left out of the mesh for lack of it, naming the assignment just made as the remedy",
|
||||
Asserts: "every line of assign's answer is derived from one resolution of the node taken after the " +
|
||||
"write committed, and agrees with plan for the node run right after the act",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay237", Files: []string{"cmd/mesh-controller/replay237_test.go"}},
|
||||
// A host without its package manager's configuration refused the declaration that would restore it
|
||||
// (2026-10-04): a declaration that left the package manager's module out removed pacman's configuration
|
||||
// file; from then on pacman did not answer, and the node-engine refused every declaration whole, the
|
||||
// one that named the module again and would have written the file back included. A person restored the
|
||||
// file by hand an hour and a half later. A test in the node-engine, written only with what it had before
|
||||
// its fix, laid over the commit before (mesh-host #90). Fix is the merge of its pull request. The
|
||||
// controller's half of the fix (mesh-controller #263, db7e4869: a machine refused or applied only partly
|
||||
// past one apply interval raises a condition) is held by its own tests there, not by this replay.
|
||||
{ID: "R233", Issue: 233, Kind: InRepository, Repository: "mesh-host", Fix: "a1f9856b3957fa0b11c04294f6367bc6febf5eb0",
|
||||
What: "the package manager's configuration file was removed, pacman stopped answering, and the node-engine " +
|
||||
"refused every declaration whole, the one that would have written the file back included",
|
||||
Asserts: "a declaration that restores the package manager's configuration is applied while pacman does not " +
|
||||
"answer: only what needs pacman fails, with the probe's words, and the file is written and recorded",
|
||||
Package: "./cmd/mesh-host", Test: "TestReplay233",
|
||||
Files: []string{"cmd/mesh-host/replay233_test.go"}},
|
||||
// Genesis pushed to a registry its runtime did not yet trust (2026-09-22): the first real genesis raised the
|
||||
// mesh's registry and pushed the control plane's image to it by the address it was given, and the runtime
|
||||
// refused plain HTTP, because the trust was written by the docker module six steps after the push. The
|
||||
// lab never saw it: its base image trusts the documentation ranges before any bed starts. A bed here,
|
||||
// whose anchor has that trust taken off before the installer runs, raised with the node-engine and its
|
||||
// installer built at the commit; the control plane genesis builds and the catalogue it reads are pinned,
|
||||
// the same on both sides.
|
||||
//
|
||||
// **Pinned to today's genesis, not to the fix's merge** (hq issues 146, 590, 596, 597): at the merge of
|
||||
// the fix (mesh-host #91, dd9ab792) no genesis got past step 6, so neither side reached the push. Fix is
|
||||
// mesh-host main with every genesis fix since; Before is the same main with #91 reverted, kept on the
|
||||
// branch replay/r92-before and never merged.
|
||||
{ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "0d11b4fc930d6723f3dc54dc173175f66fb6b3b3",
|
||||
Before: "9e233e1d87569273b4ed353c450b0bd27d21039b",
|
||||
What: "the first real genesis pushed the control plane's image to the registry it had just raised and was " +
|
||||
"refused, server gave HTTP response to HTTPS client: the runtime's trust of the registry was written " +
|
||||
"by the docker module, six steps after the push",
|
||||
Asserts: "on a runtime that trusts no registry but loopback, genesis given --registry <its own address>:5000 " +
|
||||
"pushes the control plane to the registry it raised, and the runtime trusts that registry by the " +
|
||||
"mesh's name too",
|
||||
Test: "test/integration/replay92.test.ts",
|
||||
With: []Also{
|
||||
{Repository: "mesh-controller", Fix: "49102e1645c920e42e453caf0aa0746d2b93ce90", Before: "49102e1645c920e42e453caf0aa0746d2b93ce90"},
|
||||
{Repository: "mesh-catalog", Fix: "f88e3452f8b8b285ed86e1fe93e1a153f71faada", Before: "f88e3452f8b8b285ed86e1fe93e1a153f71faada"},
|
||||
}},
|
||||
// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
|
||||
// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
|
||||
// 0240 makes the node-engine judge it and the gate fail it.
|
||||
{ID: "R-crashloop", Issue: 268, Kind: Liveness, Repository: "mesh-controller",
|
||||
Fix: "9d15f3a",
|
||||
With: []Also{{Repository: "mesh-host", Fix: "03031a4"}},
|
||||
What: "the agent server crash-looped about a hundred times while the mesh read it applied, its tools served " +
|
||||
"and nothing raised: the gate judged a module by what the mesh saw from outside",
|
||||
Asserts: "a container whose program exits at start is said unhealthy by the node-engine, and its build fails " +
|
||||
"its gate on the first machine and is put back there",
|
||||
Package: ".", Test: "TestReplayCrashLoop", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
||||
// A module's again: the silent web application, which no liveness and no TCP check sees, raised by the
|
||||
// HTTP check its module declares (ADR 0240 Phase B).
|
||||
{ID: "R145", Issue: 145, Kind: Liveness, Repository: "mesh-controller",
|
||||
Fix: "7f25666",
|
||||
With: []Also{{Repository: "mesh-host", Fix: "56e2ebe"}},
|
||||
What: "a web application accepted TCP and answered nothing for eleven hours while the mesh said its machine " +
|
||||
"healthy: its port was open and its program ran",
|
||||
Asserts: "a web server whose application never answers is said unhealthy by its declared HTTP check within two " +
|
||||
"looks, and the controller raises the module's condition on the second",
|
||||
Package: ".", Test: "TestReplaySilentWebApp", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
||||
// A merge check judged the head alone (2026-10-09): node-engine #66 was checked green at its head after #64/#65
|
||||
// had moved the trunk, both adding a field under the JSON key `declared`; git merged the two edits cleanly and the
|
||||
// trunk no longer compiled. A test in the controller's builder, written with only what it had before the fix, run
|
||||
// with a container runtime that runs each container's command on the machine. The fix is still on its branch:
|
||||
// Fix is its pull request's head, proved against the main it branched from (novox/hq ADR 0323).
|
||||
{ID: "R341", Issue: 341, Kind: InRepository, Repository: "mesh-controller", Fix: "83a9cf6", Before: "3cfc121",
|
||||
What: "node-engine #66 passed both statuses at its head and merged; merged into the trunk #65 had moved, it did " +
|
||||
"not compile: Declared redeclared",
|
||||
Asserts: "a check judges the head merged into the trunk's head, and fails a head whose merge does not compile, " +
|
||||
"saying so",
|
||||
Package: "./internal/builder", Test: "TestReplay341", Files: []string{"internal/builder/replay341_test.go"}},
|
||||
// The mesh made a value for a secret only an outside party issues (2026-10-10): nobody had given the mounts
|
||||
// module's smb-credentials, the send made a random value for it and placed it, and the module read it as a
|
||||
// credential given and wrong (ADR 0228: such a secret is never made). The fix's own test reads what the fix added,
|
||||
// so the replay is a test of its own in the controller, written with only what it had before the fix — register,
|
||||
// assign, planFor, declarationFor, the inventory's secret reads — laid over the commit before. Needs a store:
|
||||
// MESH_TEST_POSTGRES, as R237. Fix is the merge of its pull request (mesh-controller #262); Home is the commit that
|
||||
// added the replay (mesh-controller #274), reachable once that pull request merges.
|
||||
{ID: "R401", Issue: 401, Kind: InRepository, Repository: "mesh-controller", Fix: "1e28342edec170d59570e8127ddead7b19a70c20",
|
||||
What: "the send made a random value for an own secret issued outside the mesh that nobody gave, and placed it: " +
|
||||
"the module read it as a credential given and wrong",
|
||||
Asserts: "the send makes no value for an own secret issued outside the mesh that nobody gave and places none, " +
|
||||
"places no value the mesh made for it before, and places a value once a person gives one",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay401", Files: []string{"cmd/mesh-controller/replay401_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "f3c15e00af5004d053bc424fe01c7f9a24656d59"},
|
||||
// A plan's id was pasted into its query (2026-10-12): an id given to the plans verb holding a quote answered
|
||||
// another plan, or failed as a syntax error. The fix's own test is a file of its own using only what the
|
||||
// inventory had before (ForTest, SavePlan, PlanByID, RecentPlans), so it is laid over the commit before. Needs a
|
||||
// store: MESH_TEST_POSTGRES. Fix is the merge of its pull request (mesh-controller #268).
|
||||
{ID: "R561", Issue: 561, Kind: InRepository, Repository: "mesh-controller", Fix: "f8daec01d3f085ad30aa7fb584055d8d451c79dc",
|
||||
What: "a plan's id was pasted into the SQL that reads it: an id holding a quote answered another plan, or " +
|
||||
"failed as a syntax error",
|
||||
Asserts: "an id holding a quote is read as a plain id: no plan answers it but one of that very id, and " +
|
||||
"RecentPlans takes its limit as a parameter",
|
||||
Package: "./internal/inventory", Test: "TestAPlanIdHoldingAQuoteIsReadAsAPlainId",
|
||||
Files: []string{"internal/inventory/plan_id_test.go"}},
|
||||
// A plan sent the control node a controller older than the one running there (2026-10-02): two merges a minute
|
||||
// apart made two plans; the controller's plan rolled the newer controller, which widened its seat's row, and the
|
||||
// node-engine's plan then sent the build before it, whose binary refused the row. The fix's own TestReplay201
|
||||
// reads what the fix added, so the replay is a test of its own in the controller, written with only what it had
|
||||
// before — the inventory's build, gate, assignment and send records, keepingRecorded, planFor, declarationFor —
|
||||
// laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is the merge of its pull request
|
||||
// (mesh-controller #273); Home is the commit that added the replay (mesh-controller #275), reachable once that
|
||||
// pull request merges.
|
||||
{ID: "R201", Issue: 201, Kind: InRepository, Repository: "mesh-controller", Fix: "84deb530ee5afe2ffdf4d2cc65cdc355b12f8ccb",
|
||||
What: "a plan sent the control node a controller older than the one running there, whose seat row the newer " +
|
||||
"one had widened: the older binary refused the row and the mesh went quiet",
|
||||
Asserts: "a send that is no person's push carries the controller at the build its machine was last sent, not " +
|
||||
"an older record; a person's push carries the record",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay201ThePlansSend",
|
||||
Files: []string{"cmd/mesh-controller/replay201_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "45a1783ba9a43fd466a518ee85673c701223b517"},
|
||||
// A requirement could not be optional (issue 166): every soft dependency had become a hard requirement, so a
|
||||
// manifest could not say grafana runs without a data source — a mesh without influxdb could not register it,
|
||||
// and with influxdb only in the catalogue, assigning grafana installed influxdb for it. A test of its own in
|
||||
// the controller, written with only what it had before — ParseManifest, Resolve, the resolve tests' helpers —
|
||||
// laid over the commit before. Fix is the merge of its pull request (mesh-controller #281); Home is the commit
|
||||
// that added the replay (mesh-controller #287), reachable once that pull request merges.
|
||||
{ID: "R166", Issue: 166, Kind: InRepository, Repository: "mesh-controller", Fix: "4473bc7d7315bb462dcce7d6a8927bff37ecdc59",
|
||||
What: "a manifest could not say a requirement is optional: grafana, which runs without a data source, could " +
|
||||
"not be assigned on a mesh without influxdb, and influxdb only in the catalogue was installed for it",
|
||||
Asserts: "a requirement marked optional parses; with no provider running, assigning its consumer is not refused " +
|
||||
"and installs nothing for it; with a provider running elsewhere it binds",
|
||||
Package: "./internal/catalogue", Test: "TestReplay166", Files: []string{"internal/catalogue/replay166_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "b1baa8d8baf0289b76e8082b3b95d146af6a973e"},
|
||||
// A folder could not be owned by the account its module runs as (2026-10-06): the media managers' recycle-bin
|
||||
// folders were declared owned by a literal user and group, and on a machine whose puid and pgid named another
|
||||
// account the apps refused them as not writable. The fix's own tests read what the fix added, so the replay is
|
||||
// a test of its own in the controller, written with only what it had before — Compose, a setting layer, the
|
||||
// adoption tests' anchor — laid over the commit before. Fix is the merge of its pull request (mesh-controller
|
||||
// #260); Home is the commit that added the replay, reachable once its pull request merges.
|
||||
{ID: "R279", Issue: 279, Kind: InRepository, Repository: "mesh-controller", Fix: "9eddf36503f2a9af7ac22f6dffd4ab4462592b27",
|
||||
What: "the TV and film managers refused their recycle-bin folders as not writable: a declared directory's owner " +
|
||||
"was literal, and could not follow the puid and pgid the module runs as on that machine",
|
||||
Asserts: "a directory owned by ${setting:puid}:${setting:pgid} reaches the node-engine owned by the numbers the " +
|
||||
"machine's settings give",
|
||||
Package: "./internal/catalogue", Test: "TestReplay279", Files: []string{"internal/catalogue/replay279_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "3bdab29a5a0ed0f15cbfdb4738a321f968ac9467"},
|
||||
// A seat did not name the paths its holder owns (2026-10-10): who owns /etc/hosts was enforced only when two
|
||||
// modules on one machine declared it, so a module not claiming node-hostname could declare it, and a holder of
|
||||
// node-hostname could declare none of its paths. A test of its own in the controller, written with only what it
|
||||
// had before — CatalogueProblems, SeatNamed, CanHold — laid over the commit before. Fix is the merge of its pull
|
||||
// request (mesh-controller #267); Home as R279.
|
||||
{ID: "R412", Issue: 412, Kind: InRepository, Repository: "mesh-controller", Fix: "4dbd4066ca54681bd92169502379ec42ff7599b6",
|
||||
What: "a module not claiming node-hostname could declare /etc/hosts, and a module could claim node-hostname " +
|
||||
"declaring neither /etc/hosts nor /etc/hostname: nothing refused either",
|
||||
Asserts: "a module declaring a seat's path without claiming the seat is refused naming the seat, and a claim " +
|
||||
"whose module does not declare the seat's paths is refused naming each",
|
||||
Package: "./internal/catalogue", Test: "TestReplay412", Files: []string{"internal/catalogue/replay412_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "3bdab29a5a0ed0f15cbfdb4738a321f968ac9467"},
|
||||
// A verb set what every agent session obeys (2026-10-09): the agent's module keeps the managed settings and tool
|
||||
// servers every Claude Code session on a node obeys in a mergeable file, and the settings verb took any key for it
|
||||
// from any caller, so an agent could plant a SessionStart hook in the operator's sessions on every node; an
|
||||
// identity provider's empty runtime configuration took a url of the caller's the same way. The fix's own tests
|
||||
// read what the fix added, so the replay is a test of its own in the controller, written with only what it had
|
||||
// before — a mesh, a registered manifest, an assignment, the settings verb run as a verb and as the terminal runs
|
||||
// it, the inventory's layer — laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is the merge of
|
||||
// its pull request (mesh-controller #172); Home is the commit that added the replay, reachable once its pull
|
||||
// request merges.
|
||||
{ID: "R340", Issue: 340, Kind: InRepository, Repository: "mesh-controller", Fix: "4d60628f37530c23c072011a697e6014de14b3f6",
|
||||
What: "the settings verb took a SessionStart hook in the agent module's managed settings, a tool server of the " +
|
||||
"caller's, and a url for an identity provider's empty runtime configuration, from any caller of the mesh's verbs",
|
||||
Asserts: "through a verb, any change to the layer of a module with a mergeable file not marked trusted: false is " +
|
||||
"refused as the controller's terminal's and leaves the layer as it was; at the terminal it is taken",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay340", Files: []string{"cmd/mesh-controller/replay340_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "e44c726a38125ade30a6d70cee9873a4e2ea6ac2"},
|
||||
// A merge acted on late superseded the merge after it (2026-10-09): two catalogue merges were acted on in reverse
|
||||
// order. The later one, a security fix to the forge's module, was planned first; the earlier one, handed over late,
|
||||
// was planned after it, superseded its plan at tier 0 and planned the fixed module again at the commit without the
|
||||
// fix, because a plan's age was read from when it was made, never from the branch's order. The fix (mesh-controller
|
||||
// #179) ordered a branch's plans by their merge times and planned a late merge at the newer commit; ADR 0276
|
||||
// (mesh-controller #197) later replaced that with batches, and the replay holds the outcome under both. A test of its
|
||||
// own in the controller, written with only what it had before — a mesh, registered modules, following.SourceMoved,
|
||||
// the plans the inventory keeps — laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is the merge of
|
||||
// its pull request; Home is the commit that added the replay, reachable once its pull request merges.
|
||||
{ID: "R349", Issue: 349, Kind: InRepository, Repository: "mesh-controller", Fix: "9ca7952d5e0a5ed1e23bdcf5aea36fe967e62d14",
|
||||
What: "a merge handed over late superseded the plan of the merge made after it, and planned the module that " +
|
||||
"merge fixed again at the earlier commit, without the fix",
|
||||
Asserts: "with the later merge acted on first and the earlier one after it, every plan left standing that plans " +
|
||||
"the fixed module plans it at the later merge's commit, and one does",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay349", Files: []string{"cmd/mesh-controller/replay349_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "5740874ce3b99b47c27403f33e6c6d2a334ad3d7"},
|
||||
// A resolver that stopped answering ran on unnoticed, and builds sent after it were blamed for it (2026-10-09):
|
||||
// the control node's resolver refused every question on its private address from 10:57:57 UTC and the machine's
|
||||
// network condition was raised at 10:58:45. The controller and the node-engine sent at 10:59:34 failed their gate
|
||||
// with "raised since it was sent", because the restarted engine's first, undecided statement cleared the condition
|
||||
// and its next look raised it anew after the send. A test of its own in the controller, written with only what it
|
||||
// had before — a mesh, stateHealth, the keeper's open conditions, aboutTheMachine — laid over the commit before.
|
||||
// Needs a store: MESH_TEST_POSTGRES. Fix is the merge of its pull request (mesh-controller #179); Home is the
|
||||
// commit that added the replay, reachable once its pull request merges. The resolver's own half — it binds each
|
||||
// address once and its watch asks it a real question and restarts it (mesh-catalog #161, 105cc7c0) — has no
|
||||
// replay here: the module had no Go before that fix, and the binding's loss was never reproduced past the burst
|
||||
// that caused it (hq issue 348's diagnosis); dnsmasq's own tests and the controller's manifest test hold it.
|
||||
{ID: "R348", Issue: 348, Kind: InRepository, Repository: "mesh-controller", Fix: "9ca7952d5e0a5ed1e23bdcf5aea36fe967e62d14",
|
||||
What: "a network fault raised before a send was cleared by a restarted node-engine's undecided statement and " +
|
||||
"raised again after it, and the gate put back the builds sent after the fault began",
|
||||
Asserts: "a build sent while the machine's network condition is open, which flaps after the send, is not held " +
|
||||
"for it, and an undecided statement does not clear the condition",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay348AFaultFromBeforeTheSendFailsNoGate",
|
||||
Files: []string{"cmd/mesh-controller/replay348_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "6eea92c323f4862d4162f9d977c8783d8a62efc3"},
|
||||
// A credential only a person can give was made anyway (found provisioning the media machine, re-checked
|
||||
// 2026-10-08): a Servarr app's API key, an indexer's key and a downloader's password are held by the provider's
|
||||
// software, and for each consumer of them the send made a random pair credential, sealed it to both ends and
|
||||
// placed it; the provider refused it, and a consumer retrying it could be banned (ADR 0337: such a credential is
|
||||
// never made). A test of its own in the controller, written with only what it had before the fix — register,
|
||||
// assign, planFor, declarationFor, AcceptSecretForPair, SecretsOf — laid over the commit before. Needs a store:
|
||||
// MESH_TEST_POSTGRES. Fix is the merge of its pull request (mesh-controller #298), which also added the replay.
|
||||
{ID: "R164", Issue: 164, Kind: InRepository, Repository: "mesh-controller", Fix: "584aa2d7d7a5a68b5087970feb93d0b1f1d4f503",
|
||||
What: "the send made a random pair credential for a provision whose credential only a person can give, and " +
|
||||
"placed it: the provider refused it",
|
||||
Asserts: "for a provision whose offer says its credential is issued outside the mesh, the send makes no value " +
|
||||
"and places none, the consumer's machine still composes, and a value a person gave for the pair is placed",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay164", Files: []string{"cmd/mesh-controller/replay164_test.go"}},
|
||||
// A README-only merge made a new digest of an unchanged Go program (2026-10-13): a module built from its
|
||||
// repository's root is compiled with the clone's .git in view, and Go stamped the commit into the binary — the
|
||||
// node-engine on the laptop carried vcs.revision=7a41e120… — so a rebuild of an unchanged build source was a new
|
||||
// program (novox/hq ADR 0267 rule 6). The builder run whole, Build to the packed bundle, at two commits differing
|
||||
// only in README.md, with the toolchain's command run on this machine; written with only what the builder had
|
||||
// before. Fix is the merge of its pull request (mesh-controller #292); Home is the commit that added the replay,
|
||||
// reachable once its pull request merges.
|
||||
{ID: "R363", Issue: 363, Kind: InRepository, Repository: "mesh-controller", Fix: "cd6757f1e3c818bc6043178bfb4edb42b8f3c63d",
|
||||
What: "two builds of a Go module built from its repository's root, at commits differing only in its README, " +
|
||||
"made two digests: the commit was stamped into the binary",
|
||||
Asserts: "two builds of one Go module whose sources differ only outside its build source give one digest",
|
||||
Package: "./internal/builder", Test: "TestReplay363", Files: []string{"internal/builder/replay363_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "94c928b1f5af099c328eb6950550b6380df24b31"},
|
||||
// A module's name was taken over by another repository (2026-10-04): the photo app's module was registered from
|
||||
// the app's own repository, and rebuilding it "to main" built the catalogue's stub of the same name instead. The
|
||||
// controller registered the outcome, recorded the catalogue as the module's source, and the next push replaced the
|
||||
// app with the stub: two sites' containers and five routes went. The fix (mesh-controller #174, hq ADR 0266 §7)
|
||||
// registers an outcome only from the repository the module is registered from, unless asked at the controller's
|
||||
// terminal. A test of its own in the controller, written with only what it had before — a mesh, RegisterModule,
|
||||
// SourceOf, a kept build request, takeIn — laid over the commit before. Needs a store: MESH_TEST_POSTGRES. Fix is
|
||||
// the merge of its pull request; Home is the commit that added the replay, reachable once its pull request merges.
|
||||
{ID: "R239", Issue: 239, Kind: InRepository, Repository: "mesh-controller", Fix: "eb7a15808641f2075d9fd1660fa0e949b9cd7230",
|
||||
What: "a build of a module's name from another repository was registered over the module's own, and the next " +
|
||||
"push replaced the photo app with the catalogue's stub",
|
||||
Asserts: "a build asked through a verb, of a module's name from a repository other than the one it is registered " +
|
||||
"from, is refused naming both sources, and the module keeps its source and version",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay239AModuleNameIsNotTakenOverByAnotherRepository",
|
||||
Files: []string{"cmd/mesh-controller/replay239_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "95556b8def3f3f21e3d037617d73f97606e6ea1f"},
|
||||
// A new SDK rebuilt neither its toolchain nor the bundles compiled in it (2026-10-13): mesh-sdk 0.1.14 planned
|
||||
// "builds mesh-sdk, sent nowhere". A built manifest is registered without its `build`, so mesh-tools' `build.on`
|
||||
// naming the SDK's package and every bundle's toolchain were edges only before a module's first build — the
|
||||
// fixes of issues 211 and 212 lost once modules were built. The fix (mesh-controller #296) reads them from what is
|
||||
// kept: the package's name@version a build stood on, and the bundles' languages. A test of its own in the
|
||||
// controller, written with only what it had before — RegisterModule, RecordBuild, theGraph, pullScope, reasoned
|
||||
// as `delivery-plan` reasons — laid over main, where it plans [[mesh-sdk]]. Needs a store: MESH_TEST_POSTGRES.
|
||||
// Fix is the merge of its pull request.
|
||||
{ID: "R614", Issue: 614, Kind: InRepository, Repository: "mesh-controller",
|
||||
Fix: "404b4409e70ed7365bb8a53862e4b197deb0bdc6",
|
||||
What: "a new SDK was planned alone: neither the toolchain built on its package nor the TypeScript bundles " +
|
||||
"compiled in that toolchain were rebuilt, so the SDK's fix reached no machine",
|
||||
Asserts: "a change to the SDK plans the SDK, then the toolchain, then every bundle compiled in it, from modules " +
|
||||
"registered as built and builds recorded with what they stood on",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles",
|
||||
Files: []string{"cmd/mesh-controller/replay614_test.go"}},
|
||||
// A mesh-host change and a mesh-controller change of one delivery group that need each other could not both pass
|
||||
// their own merge checks (2026-10-16, issue 867): each was read beside the other's repository at the commit the mesh
|
||||
// runs, which holds neither change. The fix (mesh-controller #348, hq ADR 0382) asks mesh-delivery's groups once per
|
||||
// check and reads each sibling's repository at its merge. TestReplay867 runs the whole path — the forge's
|
||||
// announcement, the controller's ask, the build seat's check with a container runtime that runs on this machine —
|
||||
// for the pair, written with only what the controller and the build seat had before. Needs a store:
|
||||
// MESH_TEST_POSTGRES. R-issue867-recheck, R-issue867-recheck-verb and R-issue867-release-group are the three defects found live in the same fix, each with its own merge.
|
||||
{ID: "R867", Issue: 867, Kind: InRepository, Repository: "mesh-controller",
|
||||
Fix: "a47976be4e699c35681be87c48ecfedd793374cd",
|
||||
What: "a mesh-host change and a mesh-controller change of one delivery group, each needing the other, could not " +
|
||||
"both pass their own merge check: each was read beside the other at the commit the mesh runs",
|
||||
Asserts: "each member of a delivery group passes its own merge check, read beside the other member's head " +
|
||||
"merged onto its base",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay867", Files: []string{"cmd/mesh-controller/replay867_test.go"}},
|
||||
// A recheck announces the pull request without its head branch, so its group was not found and the verdict said
|
||||
// nothing of its siblings (mesh-controller #350): the group is found by repository and number.
|
||||
{ID: "R-issue867-recheck", Issue: 867, Kind: InRepository, Repository: "mesh-controller",
|
||||
Fix: "af60d2719c0c6c2f1ff44347037280d6ba5e72b4",
|
||||
What: "a recheck of a group member read no siblings: it carries no head branch name, and the group was found by it",
|
||||
Asserts: "a pull request announced as a recheck is, without its head branch, found in its group by repository " +
|
||||
"and number, and passes beside its sibling",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay867Recheck", Files: []string{"cmd/mesh-controller/replay867_test.go"}},
|
||||
// `delivery check` runs as a command of its own, where the controller is not the serving one: it asked mesh-delivery
|
||||
// on no bus ("this controller is not on the bus") and read no siblings (mesh-controller #351).
|
||||
{ID: "R-issue867-recheck-verb", Issue: 867, Kind: InRepository, Repository: "mesh-controller",
|
||||
Fix: "a07dd78c3e90335bbe06970799e9b36a142f68c9",
|
||||
What: "the recheck verb asked mesh-delivery for the group on no bus connection and read no siblings",
|
||||
Asserts: "`delivery check` with one member, on a test bus with a mesh-delivery answering groups, learns the " +
|
||||
"sibling, asking once",
|
||||
Package: "./cmd/mesh-controller", Test: "TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn",
|
||||
Files: []string{"cmd/mesh-controller/replay867_test.go"}},
|
||||
// A release-branch group's composed check took the release judge, which cannot build a pair consistent only
|
||||
// together (mesh-controller #352, hq ADR 0239 §14): the group's own controller judges, with the host member's
|
||||
// validator. Both tests: the controller member primary, and the node-engine member primary with Judge validator.
|
||||
{ID: "R-issue867-release-group", Issue: 867, Kind: InRepository, Repository: "mesh-controller",
|
||||
Fix: "d70bb1f38867fad3595c96c7f7f2deb0b3ff501a",
|
||||
What: "the composed check of a delivery group into a release branch was judged by the release branch's " +
|
||||
"controller, which does not build with the pair's validator",
|
||||
Asserts: "a group's composed check into a release branch is judged by the group's own controller with the " +
|
||||
"mesh-host member's validator, and passes",
|
||||
Package: "./cmd/mesh-controller", Test: "TestAGroupsComposition",
|
||||
Files: []string{"cmd/mesh-controller/replay867_test.go"}},
|
||||
}
|
||||
|
||||
// Find is the replay of that id, or of that issue.
|
||||
|
||||
@@ -2,6 +2,7 @@ package replays
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -16,7 +17,8 @@ func TestEveryReplayIsWhole(t *testing.T) {
|
||||
if r.Issue == 0 || r.What == "" || r.Asserts == "" || r.Repository == "" || r.Fix == "" || r.Test == "" {
|
||||
t.Errorf("%s does not say its incident, its outcome, its fix and its test: %+v", r.ID, r)
|
||||
}
|
||||
if !regexp.MustCompile(`^R\d+$`).MatchString(r.ID) {
|
||||
// By its issue, or — for a replay that is not one core incident's — by a name of its own.
|
||||
if !regexp.MustCompile(`^R(\d+|-[a-z][a-z0-9-]*)$`).MatchString(r.ID) {
|
||||
t.Errorf("%q is not a replay's id", r.ID)
|
||||
}
|
||||
switch r.Kind {
|
||||
@@ -24,6 +26,18 @@ func TestEveryReplayIsWhole(t *testing.T) {
|
||||
if len(r.Files) == 0 || r.Package == "" {
|
||||
t.Errorf("%s runs in its repository and lays no files over the commit", r.ID)
|
||||
}
|
||||
case Liveness:
|
||||
if len(r.Files) == 0 || len(r.With) == 0 {
|
||||
t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r)
|
||||
}
|
||||
case Bed:
|
||||
if !regexp.MustCompile(`^test/integration/[a-z0-9-]+\.test\.ts$`).MatchString(r.Test) {
|
||||
t.Errorf("%s is a bed and names no bed of the lab to raise: %q", r.ID, r.Test)
|
||||
}
|
||||
if !slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-controller" }) ||
|
||||
!slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-catalog" }) {
|
||||
t.Errorf("%s is a bed and does not pin the controller and the catalogue it raises: %+v", r.ID, r)
|
||||
}
|
||||
case Bus, Resolver:
|
||||
default:
|
||||
t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build integration
|
||||
|
||||
package replays
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
//go:build integration
|
||||
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq
|
||||
// ADR 0240 Phase B, "how it is checked", rule 4; issue 145).
|
||||
//
|
||||
// For eleven hours a web application's port was open and its program ran while every request hung, and the
|
||||
// mesh said its machine was healthy; a person found it. Liveness cannot see it and a TCP check cannot either.
|
||||
// The replay is the whole chain, each half at its commit:
|
||||
//
|
||||
// 1. a web server whose application never answers is raised here, as the node-engine raises a module's;
|
||||
// 2. the node-engine at MESH_REPLAY_HOST looks at it with the HTTP check its module declares, two looks,
|
||||
// and writes what it states (its TestReplaySilentWebAppIsSaidUnhealthy) — an engine older than the
|
||||
// declared check states it alive and nothing more;
|
||||
// 3. the controller at MESH_REPLAY_CONTROLLER hears that statement on two looks in a row and raises the
|
||||
// module's condition on the second (its TestReplaySilentWebAppIsRaisedWithinTwoLooks).
|
||||
//
|
||||
// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is removed
|
||||
// after, whatever happened.
|
||||
func TestReplaySilentWebApp(t *testing.T) {
|
||||
host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host"))
|
||||
controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller"))
|
||||
for _, dir := range []string{host, controller} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir)
|
||||
}
|
||||
}
|
||||
if os.Getenv("MESH_TEST_POSTGRES") == "" {
|
||||
t.Skip("no MESH_TEST_POSTGRES: the controller's half raises its condition in a store")
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the web application is a container")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 1. The silent web application (mesh-host internal/liveness SilentWebImage, SilentWebProgram).
|
||||
const image = "busybox:1.36"
|
||||
if err := docker.Pull(ctx, image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := fmt.Sprintf("mesh-replay-silent-web-%d", time.Now().UnixNano())
|
||||
id, address, err := docker.Start(ctx, Run{Image: image, Name: name, Restart: "unless-stopped",
|
||||
Labels: map[string]string{"mesh-host.id": "app.server"},
|
||||
Cmd: []string{"sh", "-c", "mkdir -p /www/cgi-bin && printf '#!/bin/sh\\nsleep 3600\\n' > /www/cgi-bin/app && " +
|
||||
"chmod +x /www/cgi-bin/app && exec httpd -f -p 8080 -h /www"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), id)
|
||||
|
||||
// 2. The node-engine at its commit looks at it, and says what it states.
|
||||
statement := filepath.Join(t.TempDir(), "statement.json")
|
||||
engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplaySilentWebAppIsSaidUnhealthy$",
|
||||
"./internal/liveness/")
|
||||
engine.Dir = host
|
||||
engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_ADDRESS="+address,
|
||||
"MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=")
|
||||
out, err := engine.CombinedOutput()
|
||||
switch {
|
||||
case err == nil && !strings.Contains(string(out), "no tests to run"):
|
||||
t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement)))
|
||||
case judgesNothing(string(out)) || strings.Contains(string(out), "no tests to run"):
|
||||
// An engine older than the declared check: it states the application alive, and nothing more.
|
||||
t.Fatalf("the node-engine at this commit does not look at a declared HTTP check")
|
||||
default:
|
||||
t.Fatalf("the node-engine did not say the silent web application unhealthy: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
|
||||
// 3. The controller at its commit raises the module's condition from what the engine said.
|
||||
raise := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplaySilentWebAppIsRaisedWithinTwoLooks$",
|
||||
"./cmd/mesh-controller/")
|
||||
raise.Dir = controller
|
||||
raise.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off")
|
||||
out, err = raise.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("the controller did not raise the silent web application within two looks: %v\n%s", err, lastOf(string(out), 12))
|
||||
}
|
||||
if strings.Contains(string(out), "no tests to run") {
|
||||
t.Fatal("the controller at this commit has no half of the replay to run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//go:build integration
|
||||
|
||||
package replays
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// THE STORE GENESIS RAISES INITIALISES WHERE THE POSTGRES MODULE KEEPS IT (novox/hq ADR 0338 §5, issue 609).
|
||||
//
|
||||
// Genesis raises the store as the catalogue's postgres module declares its server: the module's image, its
|
||||
// data in a directory made before the store starts and owned 999:70 as the module declares it, and its
|
||||
// superuser password read from a file. The measurement the decision names: the module's image, started on
|
||||
// such a directory, empty, initialises a store that answers with that password. The image is read from the
|
||||
// catalogue's manifest, so the measurement moves with the module's pin.
|
||||
//
|
||||
// Needs a container runtime (its socket) and the catalogue (MESH_REPLAY_CATALOGUE, or the checkout beside
|
||||
// the lab); skipped aloud without either.
|
||||
func TestTheStoreGenesisRaisesInitialisesOnTheDirectoryGenesisMakes(t *testing.T) {
|
||||
catalogue := os.Getenv("MESH_REPLAY_CATALOGUE")
|
||||
if catalogue == "" {
|
||||
catalogue = filepath.Join("..", "..", "mesh-catalog")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(catalogue, "modules", "postgres", "module.json"))
|
||||
if err != nil {
|
||||
t.Skipf("no catalogue to read the store's image from (%v); MESH_REPLAY_CATALOGUE names one", err)
|
||||
}
|
||||
docker, ok := DockerFromEnv()
|
||||
if !ok {
|
||||
t.Skip("no container runtime: the store runs in a container")
|
||||
}
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
ID, Type, Path, Owner, Mode, Image string
|
||||
Env map[string]string
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var image, pgdata, owner, mode string
|
||||
for _, r := range m.Resources {
|
||||
switch {
|
||||
case r.Type == "container" && r.ID == "server":
|
||||
image, pgdata = r.Image, r.Env["PGDATA"]
|
||||
case r.Type == "directory" && r.ID == "store-data":
|
||||
owner, mode = r.Owner, r.Mode
|
||||
}
|
||||
}
|
||||
if image == "" || pgdata == "" || owner == "" {
|
||||
t.Fatalf("the postgres module names no server image (%q), PGDATA (%q) or data directory owner (%q)", image, pgdata, owner)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
// Under the working directory: a check's workspace is mounted into its containers at the same path,
|
||||
// so the runtime binds what this test made.
|
||||
here, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir, err := os.MkdirTemp(here, "store-measure-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, img := range []string{"alpine:3.20", image} {
|
||||
if err := docker.Pull(ctx, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The directory as genesis makes it, and the password file as genesis writes it: as root, a helper
|
||||
// container doing what the node-engine does on the machine.
|
||||
prep, _, err := docker.Start(ctx, Run{Image: "alpine:3.20", Binds: []string{dir + ":/w"}, Cmd: []string{"sh", "-c",
|
||||
"mkdir -p /w/store /w/state && chown " + owner + " /w/store && chmod " + orMode(mode) + " /w/store && " +
|
||||
"printf measured-pw > /w/state/superuser.secret && chmod 0600 /w/state/superuser.secret"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, said, err := docker.Wait(ctx, prep); err != nil || code != 0 {
|
||||
t.Fatalf("the directory could not be made as genesis makes it: %v %d %s", err, code, said)
|
||||
}
|
||||
docker.Remove(ctx, prep)
|
||||
t.Cleanup(func() {
|
||||
clean, _, err := docker.Start(context.Background(), Run{Image: "alpine:3.20", Binds: []string{dir + ":/w"},
|
||||
Cmd: []string{"sh", "-c", "rm -rf /w/store /w/state"}})
|
||||
if err == nil {
|
||||
_, _, _ = docker.Wait(context.Background(), clean)
|
||||
docker.Remove(context.Background(), clean)
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
})
|
||||
|
||||
store, _, err := docker.Start(ctx, Run{Image: image,
|
||||
Env: []string{"POSTGRES_PASSWORD_FILE=/run/secrets/superuser", "PGDATA=" + pgdata},
|
||||
Binds: []string{dir + "/store:/var/lib/postgresql/data", dir + "/state/superuser.secret:/run/secrets/superuser:ro"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer docker.Remove(context.Background(), store)
|
||||
answered := false
|
||||
for i := 0; i < 90 && !answered; i++ {
|
||||
// Over TCP, and with the password: the temporary server initdb runs listens on the socket only.
|
||||
code, err := docker.Exec(ctx, store, "sh", "-c",
|
||||
"PGPASSWORD=measured-pw psql -h 127.0.0.1 -U postgres -tAc 'select 1' | grep -qx 1")
|
||||
answered = err == nil && code == 0
|
||||
if !answered {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
if !answered {
|
||||
t.Fatalf("%s did not initialise a store that answers on a directory made %s %s, as genesis makes it:\n%s",
|
||||
image, owner, orMode(mode), lastLinesOf(docker.Logs(ctx, store), 15))
|
||||
}
|
||||
}
|
||||
|
||||
func orMode(m string) string {
|
||||
if m == "" {
|
||||
return "0700"
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func lastLinesOf(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
# GENESIS PAST ITS BROKER STEP: the bed of novox/hq ADR 0333 §5 (issue 590).
|
||||
#
|
||||
# The installer raises the bus from its bundle, then adopts it as the `nats` module at its broker step (step
|
||||
# 18 of 24). Before ADR 0333 the bundle raised the bus under another name and on other paths than the module
|
||||
# keeps it, so adopting it would have raised a second bus beside the first, and genesis stopped there. The
|
||||
# bed runs the installer from bare to that step, looks at the bus, runs it on past the step, and looks again.
|
||||
#
|
||||
# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the
|
||||
# toolchains over it, as a bare machine does:
|
||||
# - anchor: the machine genesis raises;
|
||||
# - forge: serves the control plane's, the base's, the SDK's and the catalogue's repositories at the
|
||||
# commits under test over git's own protocol, so the installer builds exactly those and nothing on the
|
||||
# workstation listens for the lab. On the anchor itself it would be a listener, and genesis refuses a
|
||||
# machine something already listens on.
|
||||
scenario: genesis-keeps-its-bus
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
# The foundation, the registry, the forge genesis raises, the base and the modules it builds.
|
||||
memory: 8GiB
|
||||
cpus: 4
|
||||
disk: 60GiB
|
||||
|
||||
forge:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 1GiB
|
||||
cpus: 1
|
||||
disk: 10GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,42 @@
|
||||
# GENESIS on a runtime nobody told to trust a registry: the bed of replay R92 (novox/hq issue 92).
|
||||
#
|
||||
# The first real genesis pushed the control plane's image to the registry it had just raised, and the
|
||||
# container runtime refused: "server gave HTTP response to HTTPS client". The mesh's registry speaks plain
|
||||
# HTTP on purpose, and the trust that lets a runtime speak to it was written six steps after the push. The
|
||||
# lab never saw it, because its base image trusts the documentation ranges as plain-HTTP registries before
|
||||
# any bed starts (src/lifecycle/base.ts). The bed takes that trust back off the anchor before genesis runs,
|
||||
# so the anchor's runtime is a bare machine's: it trusts loopback and nothing else.
|
||||
#
|
||||
# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the
|
||||
# registry over it, as a bare machine does:
|
||||
# - anchor: the machine genesis raises, given its registry at its own address, 192.0.2.10:5000;
|
||||
# - forge: serves the control plane's repository at the commit under test over git's own protocol, so the
|
||||
# installer builds exactly that and nothing on the workstation listens for the lab. On the anchor
|
||||
# itself it would be a listener, and genesis refuses a machine something already listens on.
|
||||
scenario: genesis-untrusted
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
# Genesis-single's anchor: the foundation, the registry, and two control planes during the pivot.
|
||||
memory: 8GiB
|
||||
cpus: 4
|
||||
disk: 40GiB
|
||||
|
||||
forge:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
disk: 15GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -0,0 +1,33 @@
|
||||
# A machine joins through the tunnel, with the bus closed to it (novox/hq ADR 0169).
|
||||
#
|
||||
# The anchor raises the foundation from the bundle, joins over its own loopback and becomes the hub.
|
||||
# The joiner has a host and nothing else: it makes its tunnel key, a token is issued for that key,
|
||||
# and it enrols. The bus is closed to the joiner's own address on the anchor, so the only way its
|
||||
# enrolment can arrive is over the tunnel the token gave it.
|
||||
scenario: joins-through-the-tunnel
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
joiner:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 1GiB
|
||||
# It carries none of the images: it only has to join.
|
||||
images: []
|
||||
|
||||
images:
|
||||
- mesh-controller:development
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
@@ -30,6 +30,9 @@ machines:
|
||||
memory: 2GiB
|
||||
|
||||
images:
|
||||
# No packet-filter runtime: the filter module's verbs are a tools bundle the node's runtime launches
|
||||
# (novox/hq ADR 0170, 0193), not a container of its own, and what this walk needs of the module is
|
||||
# the ruleset it loads — which its resources do. Stocking one built an image nothing ran.
|
||||
- mesh-controller:development
|
||||
# And the builder, because it is a module the mesh assigns rather than a program somebody
|
||||
# starts by hand — which is the only way its credential can be one the mesh delivered.
|
||||
|
||||
+115
-57
@@ -1,10 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's
|
||||
# compiled code, which serves that module's tools and runs its events/provisioner under the module's
|
||||
# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module.
|
||||
# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built
|
||||
# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193).
|
||||
#
|
||||
# build-module-runtime.sh <module> <output.tar>
|
||||
# -> tags mesh-runtime-<module>:development and saves it to <output.tar>
|
||||
#
|
||||
# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that
|
||||
# imported every module's compiled code into one process, and this script copied its dist and its
|
||||
# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/`
|
||||
# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child
|
||||
# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes
|
||||
# beside each compiled entrypoint an executable `<entry>.serve.mjs` that imports it and serves what it
|
||||
# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since,
|
||||
# so the old script failed at its first step.
|
||||
#
|
||||
# This does what the mesh's builder does for a TypeScript bundle, on the workstation:
|
||||
# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an
|
||||
# entrypoint lands where it is named);
|
||||
# 2. writes each entrypoint's launcher, executable;
|
||||
# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies;
|
||||
# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the
|
||||
# module's tools, events and provisioner entrypoints, whichever exist.
|
||||
#
|
||||
# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour,
|
||||
# and is not repeated here.
|
||||
#
|
||||
# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it
|
||||
# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of
|
||||
# other modules, it is not one). So the container is given the node's runtime credential (the node-tools
|
||||
# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served
|
||||
# module's own.
|
||||
#
|
||||
# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and
|
||||
# MESH_CATALOG checkouts. Each defaults to the sibling of this repository.
|
||||
set -euo pipefail
|
||||
|
||||
MODULE="${1:?usage: build-module-runtime.sh <module> <output.tar>}"
|
||||
@@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)"
|
||||
MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}"
|
||||
MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}"
|
||||
MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}"
|
||||
MOD="$MESH_CATALOG/modules/$MODULE"
|
||||
# Absolute, because the steps below run from inside the module.
|
||||
for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do
|
||||
[ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; }
|
||||
printf -v "$v" '%s' "$(cd "${!v}" && pwd)"
|
||||
done
|
||||
# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either.
|
||||
if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi
|
||||
MOD="$MESH_CATALOG/$MODULE"
|
||||
TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}"
|
||||
BASE="${RUNTIME_BASE:-node:22-bookworm-slim}"
|
||||
RUNTIME_SRC="$MESH_TOOLS/node-tools"
|
||||
[ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; }
|
||||
[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; }
|
||||
command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; }
|
||||
|
||||
# The SDK, built: the module compiles against its types and the launchers import its stdio loop.
|
||||
[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent )
|
||||
( cd "$MESH_SDK" && npm run build >/dev/null )
|
||||
( cd "$MESH_TOOLS" && npm run build >/dev/null )
|
||||
# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools,
|
||||
# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints
|
||||
# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage
|
||||
# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls
|
||||
# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient
|
||||
# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the
|
||||
# ambient declaration is in the program even though the dep is only installed into the image below.
|
||||
SRCS=(); for f in \
|
||||
client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
|
||||
adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \
|
||||
pg.d.ts; do
|
||||
[ -f "$MOD/$f" ] && SRCS+=("$f")
|
||||
|
||||
# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the
|
||||
# builder reads them; a module declaring none falls back to the ones the runtime serves.
|
||||
ENTRIES_JS="$(node -e '
|
||||
const m = require(process.argv[1]);
|
||||
const out = new Set();
|
||||
for (const a of (m.build && m.build.artifacts) || [])
|
||||
if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e);
|
||||
process.stdout.write([...out].join(" "));
|
||||
' "$MOD/module.json")"
|
||||
if [ -z "$ENTRIES_JS" ]; then
|
||||
for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e"
|
||||
done
|
||||
fi
|
||||
SRCS=(); for e in $ENTRIES_JS; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; }
|
||||
SRCS+=("${e%.js}.ts")
|
||||
done
|
||||
[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; }
|
||||
# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's
|
||||
# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image.
|
||||
for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done
|
||||
|
||||
# The module compiles against the SDK, which its package.json names and nothing installs: a module
|
||||
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
|
||||
# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
|
||||
# the types; the image takes the SDK from MESH_SDK below.
|
||||
# as a package copy from the sibling checkout (never a link) when absent.
|
||||
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
|
||||
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|
||||
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
|
||||
fi
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
|
||||
|
||||
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
||||
cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
|
||||
cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules"
|
||||
# And the sdk, from the sibling this script just built, whatever form the installed tree holds it
|
||||
# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised —
|
||||
# true only on a workstation where somebody had linked them, and false the moment the runtime's
|
||||
# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing
|
||||
# compiled in it. The image built then looked fine and every entry point inside it pointed at
|
||||
# nothing.
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
mkdir -p "$STAGE/node_modules/@novox"
|
||||
cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules"
|
||||
# **The image runs compiled code and never compiles any**, so it does not need a compiler. The
|
||||
# tree copied above is the runtime's full install, development dependencies and all — and the
|
||||
# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing:
|
||||
# tsc runs on the workstation a few lines above, not in here.
|
||||
#
|
||||
# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one
|
||||
# of them a git URL with no registry behind it — and could drop something the image needs.
|
||||
rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types"
|
||||
mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist"
|
||||
cp "$MESH_TOOLS/package.json" "$STAGE/package.json"
|
||||
DIST="$STAGE/modules/$MODULE/dist"
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at
|
||||
# the module, as the builder compiles, so tools/index.ts lands at tools/index.js.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"
|
||||
( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \
|
||||
--rootDir . --outDir "$DIST" 1>&2 )
|
||||
# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does.
|
||||
printf '{"type":"module","private":true}\n' > "$DIST/package.json"
|
||||
|
||||
# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder,
|
||||
# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio.
|
||||
for e in $ENTRIES_JS; do
|
||||
[ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; }
|
||||
launcher="$DIST/${e%.js}.serve.mjs"
|
||||
cat > "$launcher" <<LAUNCHER
|
||||
#!/usr/bin/env node
|
||||
// Written as the mesh's builder writes it (novox/hq ADR 0193): serve what $e registers,
|
||||
// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.
|
||||
import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";
|
||||
await import("./$(basename "$e")");
|
||||
await serveRegisteredOverStdio();
|
||||
LAUNCHER
|
||||
chmod 0755 "$launcher"
|
||||
done
|
||||
|
||||
# The SDK, from the sibling just built, as a package copy: its package.json and its dist. It has no
|
||||
# runtime dependencies of its own.
|
||||
mkdir -p "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
cp "$MESH_SDK/package.json" "$STAGE/node_modules/@novox/mesh-sdk/"
|
||||
cp -r "$MESH_SDK/dist" "$STAGE/node_modules/@novox/mesh-sdk/dist"
|
||||
|
||||
# A module may declare its own third-party runtime deps (the anthropic-manager seals with
|
||||
# tweetnacl-sealedbox-js). The shared node_modules copied above carries the common packages and
|
||||
# @novox/* — but not a module's private deps. Install those under the module itself, so Node
|
||||
# resolves them from /app/modules/<module>/node_modules and still falls back to the shared tree
|
||||
# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a
|
||||
# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.)
|
||||
MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')"
|
||||
# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from
|
||||
# /app/modules/<module>/node_modules and still falls back to /app/node_modules for the SDK. @novox/* are
|
||||
# excluded: there is no public registry for them, and the SDK is staged above.
|
||||
MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")"
|
||||
if [ -n "$MOD_DEPS" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null
|
||||
fi
|
||||
|
||||
# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist.
|
||||
ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e"
|
||||
# The runtime itself: node-tools, static, from mesh-tools' Go module.
|
||||
( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools )
|
||||
|
||||
# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the
|
||||
# module has — each by its launcher, as <module>=<path>. The others (bootstrap, prepare, apply, …) are
|
||||
# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for
|
||||
# every entrypoint.
|
||||
SERVED=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs"
|
||||
done
|
||||
|
||||
# A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio
|
||||
@@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" <<DOCKER
|
||||
FROM $BASE
|
||||
WORKDIR /app
|
||||
$EXTRA
|
||||
COPY package.json ./
|
||||
COPY node-tools /usr/local/bin/node-tools
|
||||
COPY node_modules ./node_modules
|
||||
COPY dist ./dist
|
||||
COPY modules ./modules
|
||||
ENV MESH_TOOL_MODULES=$ENTRIES
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
ENV MESH_TOOL_MODULES=$SERVED
|
||||
ENTRYPOINT ["/usr/local/bin/node-tools", "serve"]
|
||||
DOCKER
|
||||
docker build -t "$TAG" "$STAGE"
|
||||
docker save -o "$OUT" "$TAG"
|
||||
echo "built $TAG (entrypoints: $ENTRIES) -> $OUT"
|
||||
echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"
|
||||
|
||||
+18
-1
@@ -20,7 +20,24 @@ import { around, log, shorten } from "../log.ts";
|
||||
* predates the group grant cannot reach it — which is a real thing that happens on the
|
||||
* machine that just installed it.
|
||||
*/
|
||||
const INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
|
||||
let INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
|
||||
|
||||
/**
|
||||
* Point every call at another program, and say what it was pointed at before.
|
||||
*
|
||||
* **For a test about how the lab reads a failure, never about the hypervisor.** The command was
|
||||
* read from `MESH_LAB_INCUS` once, when this module loaded — and a test file setting that variable
|
||||
* in its body set it too late, because an ES module's imports are evaluated before its body. So the
|
||||
* enumeration tests asked the real incus wherever one was installed and reachable: they passed on a
|
||||
* machine without incus (the spawn fails) and failed on the workstation that runs the lab, where
|
||||
* listing succeeds. Injected here, a test names the failing program and gets it, on every machine.
|
||||
*/
|
||||
export function useIncusCommand(argv: string[]): string[] {
|
||||
if (argv.length === 0 || !argv[0]) throw new Error("an incus command needs a program to run");
|
||||
const was = INCUS;
|
||||
INCUS = [...argv];
|
||||
return was;
|
||||
}
|
||||
|
||||
export interface IncusResult {
|
||||
stdout: string;
|
||||
|
||||
+19
-3
@@ -24,6 +24,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
import type { Outcome, Skip } from "./outcomes.ts";
|
||||
import { repositories } from "./repos.ts";
|
||||
|
||||
/** What a run was taken against, per repository. */
|
||||
@@ -38,6 +39,11 @@ export interface Receipt {
|
||||
against: Against;
|
||||
/** The test files this run was pointed at. See {@link endToEnd}. */
|
||||
ran: string[];
|
||||
/** How many tests skipped, and each with its reason (novox/hq issue 693). Absent on older receipts. */
|
||||
skipped?: number;
|
||||
skips?: Skip[];
|
||||
/** Why the run did not pass although nothing failed — no test passed, or one was not reached. */
|
||||
notPassed?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -104,6 +110,7 @@ export function record(
|
||||
ran: string[],
|
||||
env = process.env,
|
||||
builtAgainst?: Against,
|
||||
outcome?: Outcome,
|
||||
): Receipt {
|
||||
const receipt: Receipt = {
|
||||
at: new Date().toISOString(),
|
||||
@@ -115,6 +122,8 @@ export function record(
|
||||
// never in the binaries. A verdict is only worth something attributed to one exact state.
|
||||
against: builtAgainst ?? whatWasTested(env),
|
||||
ran,
|
||||
...(outcome ? { skipped: outcome.skipped, skips: outcome.skips } : {}),
|
||||
...(outcome?.notPassed ? { notPassed: outcome.notPassed } : {}),
|
||||
};
|
||||
const path = receiptPath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
@@ -166,9 +175,14 @@ export function judge(
|
||||
|
||||
const days = (now.getTime() - Date.parse(receipt.at)) / 86_400_000;
|
||||
const lines: string[] = [];
|
||||
// A run that proved nothing did not pass, though nothing failed (novox/hq issue 693): no test passed,
|
||||
// or one skipped as not reached. A receipt from before the field is judged by its count alone.
|
||||
const notPassed = receipt.failed > 0 ? "" : receipt.notPassed ?? (receipt.passed === 0 ? "no test passed" : "");
|
||||
const outcome = receipt.failed > 0
|
||||
? `last ran ${ago(days)} and ${receipt.failed} test(s) failed`
|
||||
: `last passed ${ago(days)}, ${receipt.passed} test(s)`;
|
||||
: notPassed
|
||||
? `last ran ${ago(days)} and did not pass: ${notPassed}`
|
||||
: `last passed ${ago(days)}, ${receipt.passed} test(s)`;
|
||||
lines.push(`the end-to-end suite ${outcome}`);
|
||||
|
||||
let moved = false;
|
||||
@@ -192,7 +206,9 @@ export function judge(
|
||||
lines.push(` That run did not include ${endToEnd}, so it raised no machines.`);
|
||||
}
|
||||
|
||||
if (receipt.failed > 0) {
|
||||
for (const s of receipt.skips ?? []) lines.push(` skipped: ${s.test} — ${s.reason || "no reason given"}`);
|
||||
|
||||
if (receipt.failed > 0 || notPassed) {
|
||||
lines.push(" Nothing has been proven end to end since.");
|
||||
} else if (moved) {
|
||||
lines.push(" What it proved was proven about code that has since changed.");
|
||||
@@ -201,7 +217,7 @@ export function judge(
|
||||
}
|
||||
|
||||
return {
|
||||
current: endToEndRan && receipt.failed === 0 && !moved && days <= staleAfterDays,
|
||||
current: endToEndRan && receipt.failed === 0 && !notPassed && !moved && days <= staleAfterDays,
|
||||
lines,
|
||||
};
|
||||
}
|
||||
|
||||
+102
-9
@@ -23,6 +23,88 @@ export const UPSTREAM_IMAGE = "images:archlinux/current";
|
||||
|
||||
const BUILDER = "mesh-lab-base-builder";
|
||||
|
||||
/**
|
||||
* **The mirrors the builder installs from, several and in order** (novox/hq issue 700).
|
||||
*
|
||||
* The stock image names one mirror, mirrors.kernel.org, and a base build failed twice on it in one morning:
|
||||
* timeouts while installing the runtime, then a 500 on wireguard-tools. pacman tries the next server in the
|
||||
* list for each file it could not fetch, so a list of several turns one mirror's bad minute into a slower
|
||||
* download instead of a failed build.
|
||||
*
|
||||
* A fixed list rather than ranking at build time: ranking (rankmirrors, reflector) needs a package installed
|
||||
* first, from the one mirror that is the problem, and gives a different list on every build. These are
|
||||
* Arch's own geo-routed mirror first, then well-kept European mirrors near the lab's machine, then the
|
||||
* stock image's own. Each answered for core.db on 2026-10-14. A mirror that goes away costs a fallback,
|
||||
* not a build, and is replaced here.
|
||||
*/
|
||||
export const MIRRORS = [
|
||||
"https://geo.mirror.pkgbuild.com/$repo/os/$arch",
|
||||
"https://mirror.netcologne.de/archlinux/$repo/os/$arch",
|
||||
"https://ftp.fau.de/archlinux/$repo/os/$arch",
|
||||
"https://archlinux.mirror.liteserver.nl/$repo/os/$arch",
|
||||
"https://mirror.leaseweb.net/archlinux/$repo/os/$arch",
|
||||
"https://mirrors.kernel.org/archlinux/$repo/os/$arch",
|
||||
];
|
||||
|
||||
/** The mirrorlist the builder is given: every mirror, in order. */
|
||||
export function mirrorlist(): string {
|
||||
return "# Written by the mesh-lab base build (novox/hq issue 700): several mirrors, tried in order.\n" +
|
||||
MIRRORS.map((m) => `Server = ${m}`).join("\n") + "\n";
|
||||
}
|
||||
|
||||
/** The mirrors a failed pacman step names: "from <host>" in what it said, each once. */
|
||||
export function failedMirrors(said: string): string[] {
|
||||
const hosts = [...said.matchAll(/\bfrom ([a-z0-9.-]+\.[a-z]{2,})\b/gi)].map((m) => m[1]!.toLowerCase());
|
||||
return [...new Set(hosts)];
|
||||
}
|
||||
|
||||
/** How long to wait before each retry of a failed pacman step: three retries, backing off. */
|
||||
export const PACMAN_BACKOFF_MS = [10_000, 30_000, 90_000];
|
||||
|
||||
/** What a base build runs through, so a test can stand in for the hypervisor and the clock. */
|
||||
export interface BaseDeps {
|
||||
incus: (args: string[], timeoutMs?: number) => Promise<unknown>;
|
||||
incusOk: (args: string[], timeoutMs?: number) => Promise<string | null>;
|
||||
succeeds: (args: string[], timeoutMs?: number) => Promise<boolean>;
|
||||
sleep: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
const realDeps: BaseDeps = {
|
||||
incus, incusOk, succeeds, sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
};
|
||||
|
||||
/**
|
||||
* pacmanIn runs one pacman step in the builder, and **runs it again after a pause when it fails**, up to
|
||||
* three more times, saying each time which mirror failed and what it said. Installing is idempotent — what is
|
||||
* installed stays installed — so a retry repeats only what did not happen. The last failure is thrown, naming
|
||||
* every mirror that failed along the way.
|
||||
*/
|
||||
export async function pacmanIn(name: string, args: string[], log: (m: string) => void, deps: BaseDeps = realDeps):
|
||||
Promise<void> {
|
||||
const failed: string[] = [];
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
await deps.incus(["exec", name, "--", "pacman", ...args], 600_000);
|
||||
return;
|
||||
} catch (err) {
|
||||
// What pacman said, where incus kept it apart; otherwise the whole message.
|
||||
const said = (err as { stderr?: string }).stderr?.trim() || (err as Error).message;
|
||||
const mirrors = failedMirrors(said);
|
||||
for (const m of mirrors) if (!failed.includes(m)) failed.push(m);
|
||||
const where = mirrors.length > 0 ? `from ${mirrors.join(", ")}` : "naming no mirror";
|
||||
const first = said.split("\n").find((l) => /error/i.test(l))?.trim() ?? said.split("\n")[0]?.trim() ?? "";
|
||||
if (attempt >= PACMAN_BACKOFF_MS.length) {
|
||||
throw new BaseImageError(`pacman ${args.join(" ")} failed ${attempt + 1} times; the mirrors that failed: ` +
|
||||
`${failed.join(", ") || "none named"}. Last: ${said}`);
|
||||
}
|
||||
const wait = PACMAN_BACKOFF_MS[attempt]!;
|
||||
log(` pacman ${args.join(" ")} failed ${where} (${first}); trying again in ${wait / 1000}s ` +
|
||||
`(attempt ${attempt + 2} of ${PACMAN_BACKOFF_MS.length + 1})`);
|
||||
await deps.sleep(wait);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class BaseImageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
@@ -38,7 +120,9 @@ export class BaseImageError extends Error {
|
||||
*/
|
||||
export async function buildBaseImage(
|
||||
log: (message: string) => void = () => {},
|
||||
deps: BaseDeps = realDeps,
|
||||
): Promise<{ alias: string; runtime: string }> {
|
||||
const { incus, incusOk, succeeds } = deps;
|
||||
await succeeds(["delete", "-f", BUILDER], 120_000);
|
||||
|
||||
log(` launching ${BUILDER} from ${UPSTREAM_IMAGE}, with a network`);
|
||||
@@ -50,10 +134,19 @@ export async function buildBaseImage(
|
||||
], 300_000);
|
||||
|
||||
try {
|
||||
await waitForAgent(BUILDER);
|
||||
await waitForAgent(BUILDER, deps);
|
||||
|
||||
// Several mirrors before anything is fetched (MIRRORS), and read back: a list that was not written is
|
||||
// the one mirror again, and the failure would look like the one this prevents.
|
||||
log(` giving pacman ${MIRRORS.length} mirrors`);
|
||||
await incus(["exec", BUILDER, "--", "sh", "-c", `printf '%s' '${mirrorlist()}' > /etc/pacman.d/mirrorlist`], 60_000);
|
||||
const written = await incusOk(["exec", BUILDER, "--", "cat", "/etc/pacman.d/mirrorlist"], 60_000);
|
||||
if (written?.trim() !== mirrorlist().trim()) {
|
||||
throw new BaseImageError(`the mirrorlist written into ${BUILDER} reads back as ${JSON.stringify(written)}`);
|
||||
}
|
||||
|
||||
log(" installing a container runtime");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-Sy", "--noconfirm", "docker"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-Sy", "--noconfirm", "docker"], log, deps);
|
||||
|
||||
// And the tools for the private network, for the same reason as the runtime: a sealed
|
||||
// scenario cannot install them, so a lab that omits them cannot test connectivity at all —
|
||||
@@ -63,27 +156,27 @@ export async function buildBaseImage(
|
||||
// declaration; a lab that brought the interface up itself would be testing its own setup
|
||||
// rather than the mesh's.
|
||||
log(" installing the tools for the private network");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "wireguard-tools"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "wireguard-tools"], log, deps);
|
||||
|
||||
// And git, for the same reason again: a machine that builds modules clones them, and a sealed
|
||||
// scenario cannot install it. On a real build machine the mesh installs it as a package like
|
||||
// anything else — the lab is the special case, because its machines reach no mirror.
|
||||
log(" installing git, so a machine can build modules");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "git"], log, deps);
|
||||
|
||||
// And nftables, because the mesh computes a machine's filtering and delivers it as a file
|
||||
// that a service reflects — and neither the file nor the service can install what loads it.
|
||||
// Installed and NOT enabled: whether a machine filters is the mesh's decision, and a lab that
|
||||
// turned it on itself would be testing its own setup.
|
||||
log(" installing nftables, so a machine can enforce what the mesh computed");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "nftables"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "nftables"], log, deps);
|
||||
|
||||
// And dnsmasq, because a service is named under the machine it runs on — postgres.novox.internal
|
||||
// — and only a resolver can answer a name the mesh was never told about. Installed and NOT
|
||||
// started: whether a machine resolves for the mesh is the mesh's decision, and a lab that
|
||||
// turned it on itself would be testing its own setup.
|
||||
log(" installing dnsmasq, so a machine can answer names under another machine");
|
||||
await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "dnsmasq"], 600_000);
|
||||
await pacmanIn(BUILDER, ["-S", "--noconfirm", "dnsmasq"], log, deps);
|
||||
|
||||
// Trust the documentation ranges as plain-HTTP registries.
|
||||
//
|
||||
@@ -216,10 +309,10 @@ export async function baseImageExists(): Promise<boolean> {
|
||||
* Wait for the guest agent, because `launch` returning means the VM started, not that anything
|
||||
* inside it will answer.
|
||||
*/
|
||||
async function waitForAgent(name: string): Promise<void> {
|
||||
async function waitForAgent(name: string, deps: BaseDeps = realDeps): Promise<void> {
|
||||
for (let i = 0; i < 90; i++) {
|
||||
if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
if (await deps.succeeds(["exec", name, "--", "true"], 10_000)) return;
|
||||
await deps.sleep(2_000);
|
||||
}
|
||||
throw new BaseImageError(`${name} started and its agent never answered.`);
|
||||
}
|
||||
|
||||
+13
-1
@@ -135,6 +135,18 @@ async function createNetwork(
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* The uplink's own address and prefix: incus picks a free one, unless `MESH_LAB_UPLINK_V4` names it.
|
||||
*
|
||||
* **A workstation behind a VPN client may have no free range left to pick.** A corporate client that
|
||||
* routes all of 10.0.0.0/8 and 172.16.0.0/12 leaves incus refusing with "failed to automatically find
|
||||
* an unused IPv4 subnet", and no scenario can be raised at all. Naming one the host does not route —
|
||||
* `192.168.231.1/24` — is the way past it; it is the host's fact, never the scenario's.
|
||||
*/
|
||||
export function uplinkAddress(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return env["MESH_LAB_UPLINK_V4"]?.trim() || "auto";
|
||||
}
|
||||
|
||||
/**
|
||||
* The one network the lab supplies rather than the declaration.
|
||||
*
|
||||
@@ -153,7 +165,7 @@ async function createUplink(instanceId: string): Promise<string> {
|
||||
if (await succeeds(["network", "show", name], 15_000)) return name;
|
||||
await incus([
|
||||
"network", "create", name,
|
||||
"ipv4.address=auto",
|
||||
`ipv4.address=${uplinkAddress()}`,
|
||||
"ipv4.nat=true",
|
||||
"ipv6.address=none",
|
||||
`user.mesh-lab.instance=${instanceId}`,
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* What a suite's tests did, beyond the runner's totals: which were skipped and why, and whether the run
|
||||
* proved anything (novox/hq issue 693).
|
||||
*
|
||||
* **A run in which no test passed has proven nothing, whatever its exit code.** The runner exits 0 when
|
||||
* every test skipped, and the lab recorded such a run as passed: the genesis bed stopped before its
|
||||
* subject, skipped its only test as "NOT REACHED", and the run read "0 passed, 0 failed", passed.
|
||||
* Failure must be loud, and a green run that verified nothing is the quietest failure there is.
|
||||
*
|
||||
* The default export is a reporter for node's test runner (`--test-reporter`): it writes one JSON line per
|
||||
* skipped test, with its file, its name and the reason it gave, word for word. The reasons come from the
|
||||
* runner's events rather than from the spec reporter's text, because a reason runs over several lines and a
|
||||
* later test's output lands between them.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
/** One skipped test and the reason it gave ("" when it gave none). */
|
||||
export interface Skip {
|
||||
file: string;
|
||||
test: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** What a receipt keeps beside the totals: the skips, and why the run did not pass when it did not. */
|
||||
export interface Outcome {
|
||||
skipped: number;
|
||||
skips: Skip[];
|
||||
notPassed?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The words a bed's test skips with when the run stopped before its subject: the subject was not reached,
|
||||
* so nothing about it was proven (the beds say "<what> NOT REACHED: <where it stopped>").
|
||||
*/
|
||||
export const NOT_REACHED = "NOT REACHED";
|
||||
|
||||
/**
|
||||
* Whether a run passed, and if not, why — the design limit of issue 693, in one place:
|
||||
*
|
||||
* - a test failed: not passed (the runner already says so);
|
||||
* - **no test passed**: not passed, whatever was skipped — a run that skipped everything, or ran
|
||||
* nothing, verified nothing;
|
||||
* - **a test skipped as NOT REACHED**: not passed, even beside tests that passed — its subject was not
|
||||
* reached, which is the bed failing to get where it was sent, not a skip anyone declared expected;
|
||||
* - otherwise passed, its other skips listed with their reasons: a test skipped for a reason its file
|
||||
* declares (no hypervisor, an optional bed not asked for) beside tests that passed is an expected
|
||||
* skip, and the run still proved what passed.
|
||||
*
|
||||
* Counts the runner did not report (null) decide nothing here: that is reportOn's "no receipt" case.
|
||||
*/
|
||||
export function judgeRun(
|
||||
counts: { passed: number | null; failed: number | null },
|
||||
skips: Skip[],
|
||||
): { passed: boolean; why: string } {
|
||||
if (counts.failed !== null && counts.failed > 0) return { passed: false, why: `${counts.failed} test(s) failed` };
|
||||
// In any case: "not reached" says the same as "NOT REACHED".
|
||||
const unreached = skips.filter((s) => s.reason.toUpperCase().includes(NOT_REACHED));
|
||||
if (counts.passed === 0) {
|
||||
return {
|
||||
passed: false,
|
||||
why: skips.length > 0
|
||||
? `no test passed: ${skips.length} skipped${unreached.length > 0 ? `, ${unreached.length} of them as ${NOT_REACHED}` : ""}`
|
||||
: "no test passed: none ran",
|
||||
};
|
||||
}
|
||||
if (unreached.length > 0) {
|
||||
return { passed: false, why: `${unreached.length} test(s) skipped as ${NOT_REACHED}: their subject was not reached` };
|
||||
}
|
||||
return { passed: true, why: "" };
|
||||
}
|
||||
|
||||
/** A reason on one line, for a log line and a record: its whitespace folded, and at most 500 characters. */
|
||||
export function oneLine(reason: string): string {
|
||||
const flat = reason.replace(/\s+/g, " ").trim();
|
||||
return flat.length > 500 ? `${flat.slice(0, 499)}…` : flat;
|
||||
}
|
||||
|
||||
/** The skips the reporter wrote to a file; none when it wrote nothing or the file cannot be read. */
|
||||
export function readSkips(path: string): Skip[] {
|
||||
let text: string;
|
||||
try {
|
||||
text = readFileSync(path, "utf8");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const skips: Skip[] = [];
|
||||
for (const line of text.split("\n")) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const s = JSON.parse(line) as Partial<Skip>;
|
||||
skips.push({ file: String(s.file ?? ""), test: String(s.test ?? ""), reason: String(s.reason ?? "") });
|
||||
} catch {
|
||||
// A torn last line, when the runner was killed mid-write: the lines before it still stand.
|
||||
}
|
||||
}
|
||||
return skips;
|
||||
}
|
||||
|
||||
interface Event {
|
||||
type: string;
|
||||
data: { name?: string; file?: string; skip?: string | boolean; details?: { type?: string } };
|
||||
}
|
||||
|
||||
/** The reporter: one JSON line per skipped test (not per skipped suite: its tests are reported each). */
|
||||
export default async function* skipsReporter(source: AsyncIterable<Event>): AsyncGenerator<string> {
|
||||
for await (const event of source) {
|
||||
if (event.type !== "test:pass" && event.type !== "test:fail") continue;
|
||||
const { skip, details } = event.data;
|
||||
if (skip === undefined || skip === false || details?.type === "suite") continue;
|
||||
const said: Skip = { file: event.data.file ?? "", test: event.data.name ?? "", reason: typeof skip === "string" ? skip : "" };
|
||||
yield `${JSON.stringify(said)}\n`;
|
||||
}
|
||||
}
|
||||
+70
-16
@@ -10,11 +10,16 @@
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { isAbsolute, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { endToEnd, record, whatWasTested } from "./lastrun.ts";
|
||||
import { judgeRun, oneLine, readSkips, type Outcome, type Skip } from "./outcomes.ts";
|
||||
import { rebuild } from "./rebuild.ts";
|
||||
|
||||
/** counted is what the runner said, or nulls when it said nothing recognisable. */
|
||||
export function counted(output: string): { passed: number | null; failed: number | null } {
|
||||
export function counted(output: string): { passed: number | null; failed: number | null; skipped: number | null } {
|
||||
// The runner's own summary lines, each on a line of its own. Anchored, so a test *named*
|
||||
// "pass 3" cannot be mistaken for the total — which is not a hypothetical worry in a suite whose
|
||||
// tests are named in sentences.
|
||||
@@ -23,13 +28,18 @@ export function counted(output: string): { passed: number | null; failed: number
|
||||
// running this against the real runner — the fixture it was first written against was output I
|
||||
// had imagined, which is a test that agrees with the mistake it was written beside.
|
||||
const plain = output.replace(/\u001b\[[0-9;]*m/g, "");
|
||||
// **The last such line, never the first** (novox/hq issue 693): the runner prints its summary after
|
||||
// everything a test printed, so a test that prints "ℹ pass 5" before it skips would otherwise be read
|
||||
// as five passes, and a run that proved nothing as passed.
|
||||
const total = (what: RegExp) => {
|
||||
const found = plain.match(what);
|
||||
return found ? Number(found[1]) : null;
|
||||
const found = [...plain.matchAll(what)];
|
||||
const last = found[found.length - 1];
|
||||
return last ? Number(last[1]) : null;
|
||||
};
|
||||
return {
|
||||
passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/m),
|
||||
failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/m),
|
||||
passed: total(/^\s*(?:\u2139|#)\s*pass\s+(\d+)\s*$/gm),
|
||||
failed: total(/^\s*(?:\u2139|#)\s*fail\s+(\d+)\s*$/gm),
|
||||
skipped: total(/^\s*(?:\u2139|#)\s*skipped\s+(\d+)\s*$/gm),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -57,16 +67,38 @@ export async function runSuite(args: string[]): Promise<number> {
|
||||
// `test/integration/canary.test.ts` is still there and still runs when it is named; it is no
|
||||
// longer raised on the way to everything else.
|
||||
|
||||
const { code, seen } = await runFiles(files);
|
||||
console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files, process.env, against)));
|
||||
return code;
|
||||
const { code, seen, skips } = await runFiles(files);
|
||||
const counts = counted(seen);
|
||||
console.log("\n" + reportOn(counts, skips, (p, f, outcome) => record(p, f, files, process.env, against, outcome)));
|
||||
return exitOf(code, counts, skips);
|
||||
}
|
||||
|
||||
/** Run some test files, passing their output through as it arrives. */
|
||||
async function runFiles(files: string[]): Promise<{ code: number; seen: string }> {
|
||||
/**
|
||||
* The suite's exit: the runner's, unless it says 0 for a run that did not pass (novox/hq issue 693). The
|
||||
* runner exits 0 when every test skipped; the lab takes this exit as the run's verdict, so a run that
|
||||
* proved nothing must not end in 0.
|
||||
*/
|
||||
export function exitOf(code: number, counts: { passed: number | null; failed: number | null }, skips: Skip[]): number {
|
||||
if (code !== 0) return code;
|
||||
if (counts.passed === null || counts.failed === null) return code;
|
||||
return judgeRun(counts, skips).passed ? 0 : 1;
|
||||
}
|
||||
|
||||
/** The reporter that writes each skipped test and its reason (outcomes.ts), by its path. */
|
||||
const skipsReporter = fileURLToPath(new URL("./outcomes.ts", import.meta.url));
|
||||
|
||||
/**
|
||||
* Run some test files, passing their output through as it arrives, and collect what was skipped and why:
|
||||
* the spec reporter to stdout as before, and the skips reporter beside it into a file of the run's own.
|
||||
*/
|
||||
async function runFiles(files: string[]): Promise<{ code: number; seen: string; skips: Skip[] }> {
|
||||
const scratch = mkdtempSync(join(tmpdir(), "mesh-lab-suite-"));
|
||||
const skipsFile = join(scratch, "skips.jsonl");
|
||||
const running = spawn(
|
||||
process.execPath,
|
||||
["--test", "--test-concurrency=1", "--experimental-strip-types",
|
||||
"--test-reporter=spec", "--test-reporter-destination=stdout",
|
||||
`--test-reporter=${skipsReporter}`, `--test-reporter-destination=${skipsFile}`,
|
||||
...files],
|
||||
{ stdio: ["inherit", "pipe", "inherit"] },
|
||||
);
|
||||
@@ -81,7 +113,14 @@ async function runFiles(files: string[]): Promise<{ code: number; seen: string }
|
||||
const code: number = await new Promise((resolve) => {
|
||||
running.on("close", (c) => resolve(c ?? 1));
|
||||
});
|
||||
return { code, seen };
|
||||
// Named as the files were given, relative to where the suite runs, rather than as the runner's absolute paths.
|
||||
const named = (file: string) => {
|
||||
const rel = isAbsolute(file) ? relative(process.cwd(), file) : file;
|
||||
return rel.startsWith("..") ? file : rel;
|
||||
};
|
||||
const skips = readSkips(skipsFile).map((s) => ({ ...s, file: named(s.file) }));
|
||||
rmSync(scratch, { recursive: true, force: true });
|
||||
return { code, seen, skips };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -92,16 +131,31 @@ async function runFiles(files: string[]): Promise<{ code: number; seen: string }
|
||||
* could check it — which is 04-ISSUES/005 in miniature, inside the fix for it.
|
||||
*/
|
||||
export function reportOn(
|
||||
counts: { passed: number | null; failed: number | null },
|
||||
write: (passed: number, failed: number) => { against: Record<string, string>; ran: string[] },
|
||||
counts: { passed: number | null; failed: number | null; skipped?: number | null },
|
||||
skips: Skip[],
|
||||
write: (passed: number, failed: number, outcome: Outcome) => { against: Record<string, string>; ran: string[] },
|
||||
): string {
|
||||
if (counts.passed === null || counts.failed === null) {
|
||||
// A run whose result could not be read is a run nobody can say anything about. Writing
|
||||
// "0 failed" because nothing said otherwise is how a green record comes to mean nothing.
|
||||
return "could not read what the runner reported; no receipt written";
|
||||
}
|
||||
const receipt = write(counts.passed, counts.failed);
|
||||
// **A skip is said, with its reason, and a run that proved nothing says NOT PASSED** (novox/hq issue
|
||||
// 693): "0 passed, 0 failed" read as a clean run. The lab module reads these lines into its record.
|
||||
const skipped = counts.skipped ?? skips.length;
|
||||
const judged = judgeRun({ passed: counts.passed, failed: counts.failed }, skips);
|
||||
const outcome: Outcome = {
|
||||
skipped,
|
||||
skips: skips.map((s) => ({ ...s, reason: oneLine(s.reason) })),
|
||||
...(judged.passed ? {} : { notPassed: judged.why }),
|
||||
};
|
||||
const receipt = write(counts.passed, counts.failed, outcome);
|
||||
const against = Object.entries(receipt.against).map(([n, c]) => `${n} ${c}`).join(", ");
|
||||
return `recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
|
||||
`${counts.failed} failed, against ${against || "nothing in git"}`;
|
||||
const lines = [`recorded: ${receipt.ran.join(", ")} — ${counts.passed} passed, ` +
|
||||
`${counts.failed} failed, ${skipped} skipped, against ${against || "nothing in git"}`];
|
||||
if (!judged.passed) lines.push(`NOT PASSED: ${judged.why}`);
|
||||
// As the runner's own spec line has it, "<test> (<file>) # <reason>": a test's name may hold any
|
||||
// punctuation, so the file in brackets is what the lab module reads the line apart by.
|
||||
for (const s of outcome.skips) lines.push(`skipped: ${s.test} (${s.file}) # ${s.reason || "no reason given"}`);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { IncusError } from "../src/incus/client.ts";
|
||||
import {
|
||||
buildBaseImage, failedMirrors, MIRRORS, mirrorlist, PACMAN_BACKOFF_MS, pacmanIn, type BaseDeps,
|
||||
} from "../src/lifecycle/base.ts";
|
||||
import { BASE_IMAGE_ALIAS } from "../src/lifecycle/place.ts";
|
||||
|
||||
/**
|
||||
* Defends novox/hq issue 700: the base build installed from the one mirror the stock image names,
|
||||
* mirrors.kernel.org, and failed twice in a morning when that mirror timed out and then answered 500.
|
||||
*/
|
||||
|
||||
const KERNEL_500 = "error: failed retrieving file 'wireguard-tools-1.0.20260223-1-x86_64.pkg.tar.zst' from " +
|
||||
"mirrors.kernel.org : The requested URL returned error: 500\nwarning: failed to retrieve some files\n" +
|
||||
"error: failed to commit transaction (failed to retrieve some files)";
|
||||
|
||||
/** A hypervisor for one base build: pacman fails `fails` times per step, then installs. */
|
||||
function aBuilder(fails: number, said = KERNEL_500) {
|
||||
const ran: string[] = [];
|
||||
const slept: number[] = [];
|
||||
let mirrorlistWritten = "";
|
||||
const tries = new Map<string, number>();
|
||||
const deps: BaseDeps = {
|
||||
incus: async (args) => {
|
||||
ran.push(args.join(" "));
|
||||
const at = args.indexOf("pacman");
|
||||
if (at >= 0) {
|
||||
const step = args.slice(at).join(" ");
|
||||
const n = (tries.get(step) ?? 0) + 1;
|
||||
tries.set(step, n);
|
||||
if (n <= fails) throw new IncusError(args, said, 1);
|
||||
}
|
||||
const printf = args.join(" ").match(/printf '%s' '([\s\S]*)' > \/etc\/pacman.d\/mirrorlist/);
|
||||
if (printf) mirrorlistWritten = printf[1]!;
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
incusOk: async (args) => {
|
||||
const c = args.join(" ");
|
||||
if (c.endsWith("cat /etc/pacman.d/mirrorlist")) return mirrorlistWritten;
|
||||
if (c.includes("{{.ServerVersion}}")) return "29.8.2";
|
||||
if (c.includes("InsecureRegistryCIDRs")) return "[127.0.0.0/8 192.0.2.0/24]";
|
||||
if (c.startsWith("image list")) return BASE_IMAGE_ALIAS;
|
||||
return "tool 1.0";
|
||||
},
|
||||
succeeds: async () => true,
|
||||
sleep: async (ms) => { slept.push(ms); },
|
||||
};
|
||||
return { deps, ran, slept };
|
||||
}
|
||||
|
||||
test("the builder is given several mirrors, the stock one last, before anything is installed", async () => {
|
||||
const hosts = MIRRORS.map((m) => new URL(m.replace("$repo", "core").replace("$arch", "x86_64")).host);
|
||||
assert.ok(new Set(hosts).size >= 4, `too few mirrors: ${hosts.join(", ")}`);
|
||||
assert.equal(hosts.at(-1), "mirrors.kernel.org");
|
||||
assert.ok(!mirrorlist().includes("'"), "the mirrorlist is written inside single quotes and holds one");
|
||||
assert.equal(mirrorlist().split("\n").filter((l) => l.startsWith("Server = ")).length, MIRRORS.length);
|
||||
|
||||
const b = aBuilder(0);
|
||||
await buildBaseImage(() => {}, b.deps);
|
||||
const wrote = b.ran.findIndex((c) => c.includes("/etc/pacman.d/mirrorlist"));
|
||||
const firstPacman = b.ran.findIndex((c) => c.includes(" pacman "));
|
||||
assert.ok(wrote >= 0 && wrote < firstPacman, "pacman ran before the mirrorlist was written");
|
||||
});
|
||||
|
||||
test("a pacman step that fails is tried again after a pause, saying which mirror failed", async () => {
|
||||
const said: string[] = [];
|
||||
const b = aBuilder(2);
|
||||
await pacmanIn("builder", ["-S", "--noconfirm", "wireguard-tools"], (m) => said.push(m), b.deps);
|
||||
assert.equal(b.ran.length, 3);
|
||||
assert.deepEqual(b.slept, PACMAN_BACKOFF_MS.slice(0, 2));
|
||||
assert.ok(PACMAN_BACKOFF_MS[1]! > PACMAN_BACKOFF_MS[0]!, "no backoff");
|
||||
assert.equal(said.length, 2);
|
||||
assert.match(said[0]!, /failed from mirrors\.kernel\.org \(error: failed retrieving file 'wireguard-tools/);
|
||||
assert.match(said[0]!, /trying again in 10s \(attempt 2 of 4\)/);
|
||||
});
|
||||
|
||||
test("a step that keeps failing stops the build, naming every mirror that failed", async () => {
|
||||
const b = aBuilder(99, "error: too many errors from mirrors.kernel.org\nerror: failed retrieving file 'x' from ftp.fau.de : 500");
|
||||
await assert.rejects(
|
||||
pacmanIn("builder", ["-Sy", "--noconfirm", "docker"], () => {}, b.deps),
|
||||
/failed 4 times; the mirrors that failed: mirrors\.kernel\.org, ftp\.fau\.de/,
|
||||
);
|
||||
assert.equal(b.ran.length, PACMAN_BACKOFF_MS.length + 1);
|
||||
});
|
||||
|
||||
test("a whole base build rides out a mirror's bad minute on every package", async () => {
|
||||
const b = aBuilder(1);
|
||||
const built = await buildBaseImage(() => {}, b.deps);
|
||||
assert.equal(built.alias, BASE_IMAGE_ALIAS);
|
||||
const pacman = b.ran.filter((c) => c.includes(" pacman "));
|
||||
assert.equal(pacman.length, 10, pacman.join("\n"));
|
||||
});
|
||||
|
||||
test("the mirrors a failure names are read from what pacman said, each once", () => {
|
||||
assert.deepEqual(failedMirrors(KERNEL_500), ["mirrors.kernel.org"]);
|
||||
assert.deepEqual(failedMirrors("Connection timed out after 10000 milliseconds\nerror: too many errors from mirrors.kernel.org"),
|
||||
["mirrors.kernel.org"]);
|
||||
assert.deepEqual(failedMirrors("error: target not found: dockr"), []);
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { busNotKept, runningFrom, streamsFrom, written, type Seen } from "./integration/bus-kept.ts";
|
||||
|
||||
/**
|
||||
* The judgement of the bed that raises genesis past its broker step (novox/hq ADR 0333 §5, issue 590):
|
||||
* exactly one bus container, it is mesh-broker-nats, a stream written before the step is still there, and
|
||||
* the first node is still connected. Held here without a lab, so the bed's verdict is the judgement's and
|
||||
* not whatever a bed happened to compare.
|
||||
*/
|
||||
|
||||
const UPSTREAM = "nats@sha256:" + "b".repeat(64);
|
||||
const MODULES = "192.0.2.10:5000/nats@sha256:" + "c".repeat(64);
|
||||
const PORTS = "127.0.0.1:8222->8222/tcp, 0.0.0.0:5671->4222/tcp";
|
||||
|
||||
function jsz(streams: Array<[string, string, number, number]>): string {
|
||||
return JSON.stringify({ account_details: [{ name: "MESH", stream_detail: streams.map(([name, created, messages, last]) =>
|
||||
({ name, created, state: { messages, last_seq: last } })) }] });
|
||||
}
|
||||
|
||||
const before: Seen = {
|
||||
running: runningFrom(`mesh-store\tpostgres@sha256:aa\t0.0.0.0:5432->5432/tcp\nmesh-broker-nats\t${UPSTREAM}\t${PORTS}\n` +
|
||||
"mesh-controller\t192.0.2.10:5000/mesh-controller@sha256:dd\t\n"),
|
||||
streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:10:00Z", 3, 7], ["EMPTY", "2026-10-13T00:10:01Z", 0, 0]])),
|
||||
};
|
||||
|
||||
const kept: Seen = {
|
||||
running: runningFrom(`mesh-store\tpostgres@sha256:aa\t0.0.0.0:5432->5432/tcp\nmesh-broker-nats\t${MODULES}\t${PORTS}\n` +
|
||||
"mesh-controller\t192.0.2.10:5000/mesh-controller@sha256:dd\t\n"),
|
||||
streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:10:00Z", 3, 9]])),
|
||||
};
|
||||
|
||||
const SAID = "broker — the foundation's bus, adopted as the nats module\n restarting mesh-broker-nats — once, onto the nats module's image";
|
||||
const HERE = "anchor here last 3s ago";
|
||||
|
||||
test("a bus adopted in place is kept: one bus, the module's image, its streams, the node connected", () => {
|
||||
assert.deepEqual(busNotKept(before, kept, HERE, SAID), []);
|
||||
});
|
||||
|
||||
test("only a stream holding something counts as written before the step", () => {
|
||||
assert.deepEqual(written(before).map((s) => s.name), ["KV_mesh-controller_lease"]);
|
||||
});
|
||||
|
||||
test("a second bus beside the first is not kept: the fault genesis had before ADR 0333", () => {
|
||||
const two: Seen = { ...kept, running: [...kept.running,
|
||||
{ name: "mesh-broker", image: UPSTREAM, ports: "0.0.0.0:5671->4222/tcp" }] };
|
||||
const wrong = busNotKept(before, two, HERE, SAID);
|
||||
assert.equal(wrong.length, 1);
|
||||
assert.match(wrong[0] as string, /2 bus containers.*mesh-broker \(/);
|
||||
});
|
||||
|
||||
test("a bus under another name, or none, is not kept", () => {
|
||||
const renamed: Seen = { ...kept, running: kept.running.map((r) => r.name === "mesh-broker-nats" ? { ...r, name: "nats-server" } : r) };
|
||||
assert.ok(busNotKept(before, renamed, HERE, SAID).some((w) => /mesh-broker-nats does not run/.test(w)));
|
||||
const none: Seen = { ...kept, running: kept.running.filter((r) => r.name !== "mesh-broker-nats") };
|
||||
assert.ok(busNotKept(before, none, HERE, SAID).some((w) => /0 bus containers/.test(w)));
|
||||
});
|
||||
|
||||
test("a bus still on the bundle's image was not adopted", () => {
|
||||
const same: Seen = { ...kept, running: before.running };
|
||||
assert.ok(busNotKept(before, same, HERE, SAID).some((w) => /still runs .* the image the bundle raised/.test(w)));
|
||||
assert.ok(busNotKept(before, kept, HERE, "broker — adopted").some((w) => /did not say the bus restarts/.test(w)));
|
||||
});
|
||||
|
||||
test("a stream gone or made again is not kept", () => {
|
||||
const gone: Seen = { ...kept, streams: [] };
|
||||
assert.ok(busNotKept(before, gone, HERE, SAID).some((w) => /KV_mesh-controller_lease .* is gone/.test(w)));
|
||||
const again: Seen = { ...kept, streams: streamsFrom(jsz([["KV_mesh-controller_lease", "2026-10-13T00:40:00Z", 1, 1]])) };
|
||||
assert.ok(busNotKept(before, again, HERE, SAID).some((w) => /was made again/.test(w)));
|
||||
});
|
||||
|
||||
test("a node the mesh has not heard from since the step is not connected", () => {
|
||||
assert.ok(busNotKept(before, kept, "anchor out of touch 4m", SAID).some((w) => /has not heard from the first node/.test(w)));
|
||||
assert.ok(busNotKept(before, kept, "", SAID).some((w) => /no record/.test(w)));
|
||||
});
|
||||
@@ -1,11 +1,14 @@
|
||||
// Set before importing: the client reads MESH_LAB_INCUS once, at module load.
|
||||
// `false` is a real program that exits non-zero and prints nothing — which is also the worst
|
||||
// case, because an empty stderr is how a failure arrives with no explanation.
|
||||
process.env["MESH_LAB_INCUS"] = "false";
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { instanceExists, taggedInstances, taggedNetworks } from "../src/incus/client.ts";
|
||||
import { instanceExists, taggedInstances, taggedNetworks, useIncusCommand } from "../src/incus/client.ts";
|
||||
|
||||
// `false` is a real program that exits non-zero and prints nothing — which is also the worst
|
||||
// case, because an empty stderr is how a failure arrives with no explanation.
|
||||
//
|
||||
// **Injected, never set in the environment here.** This file used to set MESH_LAB_INCUS in its body,
|
||||
// which runs after the imports above — so the client had already read the variable, and these tests
|
||||
// asked the real incus: green on a machine without one, red on the workstation that runs the lab.
|
||||
useIncusCommand(["false"]);
|
||||
|
||||
/**
|
||||
* "I cannot see" must never be answered as "there is nothing there."
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { carry } from "./integration/forge.ts";
|
||||
|
||||
/**
|
||||
* A bed's forge takes the lab's checkouts as they are: shallow (novox/hq issue 689). The lab module clones
|
||||
* every repository with `--depth`, and the genesis bed's forge refused the push ("shallow update not
|
||||
* allowed"), so the bed never reached its installer. Held here with a real git and no lab.
|
||||
*/
|
||||
|
||||
const git = (cwd: string, ...args: string[]) =>
|
||||
execFileSync("git", ["-C", cwd, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"],
|
||||
env: { ...process.env, GIT_AUTHOR_NAME: "lab", GIT_AUTHOR_EMAIL: "lab@example.org",
|
||||
GIT_COMMITTER_NAME: "lab", GIT_COMMITTER_EMAIL: "lab@example.org" } }).trim();
|
||||
|
||||
test("a shallow checkout, as the lab clones it, is carried into the bed's forge and served from it", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-689-"));
|
||||
try {
|
||||
const origin = join(dir, "origin");
|
||||
execFileSync("git", ["init", "-q", "-b", "main", origin]);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
writeFileSync(join(origin, "file"), `${i}\n`);
|
||||
git(origin, "add", "file");
|
||||
git(origin, "commit", "-q", "-m", `commit ${i}`);
|
||||
}
|
||||
// As the lab module clones: shallow, over a transport that honours the depth.
|
||||
const checkout = join(dir, "checkout");
|
||||
execFileSync("git", ["clone", "-q", "--depth", "2", "--branch", "main", `file://${origin}`, checkout]);
|
||||
assert.equal(git(checkout, "rev-parse", "--is-shallow-repository"), "true");
|
||||
|
||||
const bare = join(dir, "forge", "repo.git");
|
||||
const head = carry(checkout, bare, ["main", "lab"]);
|
||||
assert.equal(head, git(origin, "rev-parse", "HEAD"));
|
||||
assert.equal(git(bare, "rev-parse", "refs/heads/main"), head);
|
||||
assert.equal(git(bare, "rev-parse", "refs/heads/lab"), head);
|
||||
|
||||
// What the bed's machines do with it: clone over git's protocol, and find the commit under test.
|
||||
const served = join(dir, "served");
|
||||
execFileSync("git", ["clone", "-q", "--branch", "main", `file://${bare}`, served]);
|
||||
assert.equal(git(served, "rev-parse", "HEAD"), head);
|
||||
assert.equal(git(served, "show", "HEAD:file"), "4");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -16,7 +16,7 @@
|
||||
*
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
*
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_LAB_MODULES=<mesh-controller>/examples/modules
|
||||
* MESH_TOOLS=<mesh-tools> MESH_SDK=<mesh-sdk> (default: the checkouts beside this one)
|
||||
* MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation
|
||||
@@ -55,6 +55,7 @@ import { catalogueRoot } from "../../src/repos.ts";
|
||||
import { ready, returnTo, keep } from "../../src/warm.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts";
|
||||
import { genesis, type GenesisOptions } from "./genesis.ts";
|
||||
import { carry } from "./forge.ts";
|
||||
|
||||
const SCENARIO = "adoption";
|
||||
const CONTROL = "anchor";
|
||||
@@ -246,11 +247,7 @@ async function raiseForge(): Promise<Record<string, string>> {
|
||||
const heads: Record<string, string> = {};
|
||||
try {
|
||||
for (const [name, checkout] of Object.entries(REPOS)) {
|
||||
const bare = join(dir, `${name}.git`);
|
||||
execFileSync("git", ["init", "-q", "--bare", bare]);
|
||||
execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare,
|
||||
"HEAD:refs/heads/main", "HEAD:refs/heads/lab"], { stdio: "pipe" });
|
||||
heads[name] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||
heads[name] = carry(checkout, join(dir, `${name}.git`), ["main", "lab"]);
|
||||
}
|
||||
const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`);
|
||||
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
* in for the authenticated cross-node call a manager node would make). The node-private-key stub of
|
||||
* the earlier cut is GONE — the host uses its own real key.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* Build both runtime images into the local daemon first:
|
||||
* scripts/build-module-runtime.sh anthropic-manager /tmp/anthropic-manager.tar
|
||||
* scripts/build-module-runtime.sh anthropic-consumer /tmp/anthropic-consumer.tar
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* It needs the host binary, the foundation bundle, and the runtime image the scenario loads:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-runtime-image.sh builds mesh-runtime-audit:development into the local daemon,
|
||||
* which scenarios/audit-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
|
||||
*/
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
* would say nothing at all about removal — which is the half issue 129 asked for by name.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
* step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink.
|
||||
* ca-trust carries no image: a script, a unit, and the machine's own systemd.
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
*
|
||||
* All are assigned to the one anchor, pushed ONCE, and the node converges ONCE with every one up.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh {mongodb,unifi,postgres} build the runtime images into the local
|
||||
* daemon; scenarios/catalogue-apps.yml stocks them. mongo:7, lscr.io/linuxserver/unifi-controller
|
||||
* and synesthesiam/marytts must be in the local daemon to be stocked.
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
*
|
||||
* All is assigned to the one anchor, pushed ONCE, and the node converges ONCE with both modules up.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh {sonarr,radarr} build the runtime images into the local daemon;
|
||||
* scenarios/catalogue-media.yml stocks them. lscr.io/linuxserver/{sonarr,radarr} must be in the
|
||||
* local daemon; the service images are pulled from the internet. Each *arr runtime is given a lab
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
* the seed), and mosquitto's provisioner — running in the assigned runtime — creates a scoped client
|
||||
* for a contribution the mesh delivered, which then authenticates with the password the mesh minted.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh mosquitto builds mesh-runtime-mosquitto:development (carrying
|
||||
* mosquitto_ctrl and the compiled bootstrap entrypoint) into the local daemon, which
|
||||
* scenarios/catalogue-mqtt.yml stocks. eclipse-mosquitto:2 must be in the local daemon to be
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
* path is fulfilled by creating the consumer's login with the mesh-minted password — it seals nothing
|
||||
* and needs no seal key (novox/hq ADR 0048, issue 032-provider-runtime-has-no-seal-key).
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh {postgres,redis,minio,plex} build the four runtime images into the
|
||||
* local daemon; scenarios/catalogue-small.yml stocks them. postgres:17-alpine, redis:7-alpine and
|
||||
* minio/minio:latest is pulled from the internet by the node itself.
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
* It needs a host binary and the foundation bundle:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*
|
||||
* HELPER — stock the two runtimes into the local daemon before the run (some may already be there):
|
||||
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
* registry by digest; the host pulls and runs it on the cadence.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host (feat/apply-schedule — the scheduler that fires the step)
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* MESH_LAB_MODULES=.../mesh-controller/examples/modules (feat/schedule-container — the parser that
|
||||
* carries `schedule` through). scenarios/schedule-tick.yml stocks alpine:latest (which must be in
|
||||
* the local daemon) and serves it by digest; there is no runtime image — schedtest is a bare tick.
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
* A tool would only fail if it were actually invoked without real creds — which this bed does not do,
|
||||
* because the point is exactly that serving does not require them.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh confluence builds mesh-runtime-confluence:development into the
|
||||
* local daemon; scenarios/tools-confluence.yml stocks it. There is no service image.
|
||||
*/
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
* A tool would only fail if it were actually invoked without real creds — which this bed does not do,
|
||||
* because the point is exactly that serving does not require them.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh gitlab builds mesh-runtime-gitlab:development into the local
|
||||
* daemon; scenarios/tools-gitlab.yml stocks it. There is no service image — gitlab is tools-only.
|
||||
*/
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
* It needs a host binary and the foundation bundle:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*
|
||||
* HELPER — stock the three runtimes into the local daemon before the run (some may already be there):
|
||||
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
* The manifests are the catalogue's own (../mesh-catalog/modules/{mesh-vault,redis}/module.json), with
|
||||
* the runtime artifact named as the image the lab built, exactly as the other assigned-* beds do.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh mesh-vault / redis build the two runtime images into the local
|
||||
* daemon; scenarios/vault-node.yml stocks them.
|
||||
*/
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
* by the firewall the nftables module derives (issues 055/056/057 in one bed).
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock (the TEMPLATE)
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock (the TEMPLATE)
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
* MESH_LAB_SOURCE=git://<forge>/mesh-controller.git MESH_LAB_SOURCE_REF=<commit>
|
||||
* MESH_LAB_BUILD_REF=<branch or commit for module builds, default main>
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Whether genesis kept its bus through the broker step (novox/hq ADR 0333 §5, issue 590).
|
||||
*
|
||||
* Pure, so the judgement is tested without a lab (test/bus-kept.test.ts) and the bed only gathers what it
|
||||
* judges: the containers the anchor runs, the streams its bus holds, and what the mesh says of the node,
|
||||
* once before the broker step and once after it.
|
||||
*/
|
||||
|
||||
/** One running container, as `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'` says it. */
|
||||
export interface Running {
|
||||
name: string;
|
||||
image: string;
|
||||
ports: string;
|
||||
}
|
||||
|
||||
/** One stream on the bus, as the server's monitor says it (`/jsz?accounts=true&streams=true`). */
|
||||
export interface Stream {
|
||||
account: string;
|
||||
name: string;
|
||||
created: string;
|
||||
messages: number;
|
||||
lastSeq: number;
|
||||
}
|
||||
|
||||
/** What the anchor showed at one moment. */
|
||||
export interface Seen {
|
||||
running: Running[];
|
||||
streams: Stream[];
|
||||
}
|
||||
|
||||
/** The container the bundle raises the bus as, and the nats module keeps it as (ADR 0333 §1). */
|
||||
export const BUS = "mesh-broker-nats";
|
||||
|
||||
/** `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'`, read. */
|
||||
export function runningFrom(ps: string): Running[] {
|
||||
return ps.split("\n").map((l) => l.trim()).filter(Boolean).map((l) => {
|
||||
const [name = "", image = "", ports = ""] = l.split("\t");
|
||||
return { name, image, ports };
|
||||
});
|
||||
}
|
||||
|
||||
/** The server's `/jsz?accounts=true&streams=true`, read: every stream of every account. */
|
||||
export function streamsFrom(jsz: string): Stream[] {
|
||||
const parsed = JSON.parse(jsz) as {
|
||||
account_details?: Array<{
|
||||
name?: string;
|
||||
stream_detail?: Array<{ name?: string; created?: string; state?: { messages?: number; last_seq?: number } }>;
|
||||
}>;
|
||||
};
|
||||
const out: Stream[] = [];
|
||||
for (const account of parsed.account_details ?? []) {
|
||||
for (const s of account.stream_detail ?? []) {
|
||||
out.push({
|
||||
account: account.name ?? "", name: s.name ?? "", created: s.created ?? "",
|
||||
messages: s.state?.messages ?? 0, lastSeq: s.state?.last_seq ?? 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A bus container: one that serves the bus's port inside it, or carries a bus's name. Both, because a
|
||||
* second bus raised by a module under another name is exactly what this looks for, and so is one raised on
|
||||
* another port under the bus's name.
|
||||
*/
|
||||
export function busContainers(running: Running[]): Running[] {
|
||||
return running.filter((r) => /->4222\/tcp/.test(r.ports) || r.name.startsWith("mesh-broker"));
|
||||
}
|
||||
|
||||
/** The streams written before the step: the ones holding anything. */
|
||||
export function written(seen: Seen): Stream[] {
|
||||
return seen.streams.filter((s) => s.messages > 0 || s.lastSeq > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every way the bus was not kept, in sentences; empty when it was. `node` is the mesh's line for the first
|
||||
* node from `node list`; `said` is what the installer printed on the run that crossed the step.
|
||||
*/
|
||||
export function busNotKept(before: Seen, after: Seen, node: string, said: string): string[] {
|
||||
const wrong: string[] = [];
|
||||
const buses = busContainers(after.running);
|
||||
if (buses.length !== 1) {
|
||||
wrong.push(`${buses.length} bus containers run after the broker step, not one: ` +
|
||||
(buses.map((b) => `${b.name} (${b.image}, ${b.ports || "nothing published"})`).join("; ") || "none"));
|
||||
}
|
||||
const bus = after.running.find((r) => r.name === BUS);
|
||||
if (!bus) {
|
||||
wrong.push(`${BUS} does not run after the broker step`);
|
||||
}
|
||||
const was = before.running.find((r) => r.name === BUS);
|
||||
if (bus && was && bus.image === was.image) {
|
||||
wrong.push(`${BUS} still runs ${bus.image}, the image the bundle raised it from: the module's image never ` +
|
||||
`took its place (ADR 0333 §3)`);
|
||||
}
|
||||
if (!/restarting\s+mesh-broker-nats/.test(said)) {
|
||||
wrong.push("the installer did not say the bus restarts onto the module's image (ADR 0333 §3)");
|
||||
}
|
||||
for (const s of written(before)) {
|
||||
const now = after.streams.find((a) => a.account === s.account && a.name === s.name);
|
||||
if (!now) {
|
||||
wrong.push(`the stream ${s.name} (${s.account}), holding ${s.messages} message(s) before the step, is gone`);
|
||||
} else if (now.created !== s.created || now.lastSeq < s.lastSeq) {
|
||||
wrong.push(`the stream ${s.name} (${s.account}) was made again: created ${s.created}, last ${s.lastSeq} ` +
|
||||
`before the step; created ${now.created}, last ${now.lastSeq} after it`);
|
||||
}
|
||||
}
|
||||
if (!/^\S+\s+here\b/.test(node.trim())) {
|
||||
wrong.push(`the mesh has not heard from the first node since the step: ${node.trim() || "it has no record of it"}`);
|
||||
}
|
||||
return wrong;
|
||||
}
|
||||
@@ -17,7 +17,7 @@
|
||||
* It needs the host binary and the foundation bundle, like the mesh walk, plus a runtime image:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* MESH_LAB_RUNTIME=.../mesh-runtime-audit.tar (docker save of the runtime+audit-logger image;
|
||||
* built by scripts/build-runtime-image.sh)
|
||||
*
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* A bed's forge: a checkout's HEAD carried into a fresh bare repository, which a git daemon then serves.
|
||||
*
|
||||
* **A shallow checkout is carried as it is** (novox/hq issue 689). The lab module checks every repository
|
||||
* out shallow, and a fresh bare repository refuses a push whose history ends in a cut ("shallow update not
|
||||
* allowed"), so the genesis bed raised its anchor and forge and then skipped. The bare repository accepts
|
||||
* the cut (`receive.shallowUpdate`) and serves the history the checkout holds: the bed serves the commit
|
||||
* under test, and nothing it raises reads further back than the run's checkout does. Unshallowing first is
|
||||
* not an option in a lab run: the forge credential is gone once the run's clones are done, so a private
|
||||
* repository could not be fetched again.
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
|
||||
/**
|
||||
* Initialise `bare` and push `checkout`'s HEAD into it under each of `branches`. Returns the commit served.
|
||||
*/
|
||||
export function carry(checkout: string, bare: string, branches: string[] = ["main"]): string {
|
||||
execFileSync("git", ["init", "-q", "--bare", bare]);
|
||||
// No collection after the push: one run in the background rewrites the objects while they are carried.
|
||||
execFileSync("git", ["-C", bare, "config", "receive.autogc", "false"]);
|
||||
// The lab's checkouts are shallow, and a bare repository refuses a shallow push without this.
|
||||
execFileSync("git", ["-C", bare, "config", "receive.shallowUpdate", "true"]);
|
||||
const [first, ...rest] = branches;
|
||||
execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare, `HEAD:refs/heads/${first}`], { stdio: "pipe" });
|
||||
const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||
// The other branches are set in the bare repository itself: a shallow push of several refs at once has
|
||||
// receive-pack fail every ref after the first ("remote failed to report status").
|
||||
for (const branch of rest) execFileSync("git", ["-C", bare, "update-ref", `refs/heads/${branch}`, head]);
|
||||
return head;
|
||||
}
|
||||
@@ -31,7 +31,7 @@
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||||
* MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
* MESH_LAB_SOURCE=<forge>/mesh-controller.git MESH_LAB_SOURCE_REF=<commit>
|
||||
* MESH_LAB_KEEP=1 to leave it standing afterwards
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* GENESIS KEEPS ITS BUS through the broker step (novox/hq ADR 0333 §5, issue 590).
|
||||
*
|
||||
* The installer raises the bus from its bundle and, at its broker step (step 18 of 24), adopts it as the
|
||||
* catalogue's `nats` module. Until ADR 0333 the bundle raised the bus as `mesh-broker` on docker volumes,
|
||||
* and the module keeps it as `mesh-broker-nats` on directories of the machine: adopting it would have raised
|
||||
* a second bus beside the first, so the installer refused there and genesis stopped. ADR 0333 has the bundle
|
||||
* raise the bus where the module keeps it, so the adoption replaces the server's image once and keeps
|
||||
* everything else.
|
||||
*
|
||||
* The bed runs the installer from bare twice. The first run is given a catalogue without the `nats` module,
|
||||
* so it stops at the broker step by name, before anything is registered — the moment just before the step,
|
||||
* with everything genesis wrote on the bus by then. The bed reads the bus there: its containers, and every
|
||||
* stream holding anything. The second run, given the whole catalogue, finds the steps before done and goes
|
||||
* on through the broker step. The bed reads the bus again, and passes only when (ADR 0333 §5):
|
||||
*
|
||||
* - exactly one bus container runs, and it is mesh-broker-nats, on the module's image now;
|
||||
* - every stream that held something before the step is still there, the same stream, as long or longer;
|
||||
* - the mesh has heard from the first node since.
|
||||
*
|
||||
* The judgement is test/integration/bus-kept.ts, held without a lab by test/bus-kept.test.ts.
|
||||
*
|
||||
* **A bed that could not reach the step says so and does not fail**: a first run that stopped earlier than
|
||||
* the broker step — a pull the internet refused, a build that broke — proves nothing about the step, and the
|
||||
* test is skipped with the reason. A second run that stops AT the broker step is the fault, and fails. One
|
||||
* that stops after it has crossed it: what it stopped on is reported, and the bus is judged.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=<mesh-catalog>/modules
|
||||
* MESH_LAB_REPOS=<a directory holding mesh-controller, mesh-tools, mesh-tools-go, mesh-sdk and mesh-catalog checkouts>
|
||||
* — served by the forge machine at each checkout's HEAD; the catalogue's is the one built from
|
||||
* MESH_LAB_KEEP=1 leave it standing
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||||
import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||
import { genesis, type GenesisResult } from "./genesis.ts";
|
||||
import { carry } from "./forge.ts";
|
||||
import { BUS, busContainers, busNotKept, runningFrom, streamsFrom, written, type Seen } from "./bus-kept.ts";
|
||||
|
||||
const SCENARIO = "genesis-keeps-its-bus";
|
||||
const ANCHOR = "anchor";
|
||||
const FORGE = "forge";
|
||||
const FORGE_ADDRESS = "192.0.2.30";
|
||||
const REGISTRY = "192.0.2.10:5000";
|
||||
/** The repositories the installer clones, served by the forge at each checkout's HEAD. */
|
||||
// mesh-tools-go is found by the installer beside mesh-tools (hq issue 616): genesis builds the Go toolchain too.
|
||||
const REPOS = ["mesh-controller", "mesh-tools", "mesh-tools-go", "mesh-sdk", "mesh-catalog"] as const;
|
||||
/** The step this bed is about, as the installer names it. */
|
||||
const BROKER = "broker";
|
||||
/** The installer's steps before it, in its order: a first run stopped at one of these never reached it. */
|
||||
const BEFORE_BROKER = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry",
|
||||
"publish", "control-plane", "retire", "builder", "packages", "sdk", "base", "store"];
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const installer = bootstrapBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||
const repos = process.env["MESH_LAB_REPOS"] ?? "";
|
||||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-keeps-its-bus-live" : undefined);
|
||||
|
||||
const missingRepo = REPOS.find((r) => !repos || !existsSync(resolve(repos, r, ".git")));
|
||||
const skip =
|
||||
!capability.usable ? capability.why :
|
||||
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to the genesis bundle" :
|
||||
!catalogDir || !existsSync(join(catalogDir, "nats", "module.json")) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||
missingRepo ? `MESH_LAB_REPOS holds no ${missingRepo} checkout` :
|
||||
false;
|
||||
|
||||
let instanceId = "";
|
||||
/** Why the bed never reached the step, or "" when it did. */
|
||||
let unreached = "";
|
||||
let first: GenesisResult | undefined;
|
||||
let second: GenesisResult | undefined;
|
||||
let seenBefore: Seen | undefined;
|
||||
let seenAfter: Seen | undefined;
|
||||
let nodeLine = "";
|
||||
const heads: Record<string, string> = {};
|
||||
const report: string[] = [];
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Each repository at its checkout's HEAD, served from the forge machine over git's protocol as `main`. */
|
||||
async function raiseForge(): Promise<void> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-590-forge-"));
|
||||
try {
|
||||
for (const repo of REPOS) {
|
||||
heads[repo] = carry(resolve(repos, repo), join(dir, `${repo}.git`));
|
||||
}
|
||||
const tar = join(tmpdir(), `mesh-lab-590-forge-${process.pid}.tar`);
|
||||
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||
await push(instanceId, FORGE, tar, "/tmp/forge.tar");
|
||||
rmSync(tar, { force: true });
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
// Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing.
|
||||
await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`);
|
||||
for (let i = 0; ; i++) {
|
||||
if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-catalog.git main`)).ok) break;
|
||||
if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`);
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
}
|
||||
|
||||
/** What the anchor shows of its bus: the containers it runs, and every stream the bus holds. */
|
||||
async function look(): Promise<Seen> {
|
||||
const ps = await must(ANCHOR, `docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}'`);
|
||||
// The monitor, published on the anchor's loopback by the bundle and by the module alike.
|
||||
let jsz = "";
|
||||
for (let i = 0; ; i++) {
|
||||
const asked = await on(ANCHOR, `curl -sf --max-time 10 'http://127.0.0.1:8222/jsz?accounts=true&streams=true'`);
|
||||
if (asked.ok) { jsz = asked.out; break; }
|
||||
if (i >= 30) throw new Error(`the bus's monitor does not answer on the anchor: ${asked.out}`);
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
return { running: runningFrom(ps), streams: streamsFrom(jsz) };
|
||||
}
|
||||
|
||||
function stoppedAt(r: GenesisResult): string {
|
||||
return r.step.replace(/^step \d+ of \d+, /, "");
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||
|
||||
try {
|
||||
await raiseForge();
|
||||
report.push(` forge ${REPOS.map((r) => `${r} ${heads[r]?.slice(0, 8)}`).join(", ")}`);
|
||||
} catch (err) {
|
||||
unreached = `the forge was not raised, so the installer never ran: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`;
|
||||
const run = (catalogue: string) => genesis({
|
||||
instanceId,
|
||||
node: ANCHOR,
|
||||
installer: installer as string,
|
||||
catalogDir: catalogue,
|
||||
catalogueModules: ["distribution", "build-agent", "postgres"],
|
||||
bundleTemplate: foundationBundle(bundle, []),
|
||||
registry: REGISTRY,
|
||||
source: forge("mesh-controller"),
|
||||
sourceRef: heads["mesh-controller"] as string,
|
||||
toolsSource: forge("mesh-tools"),
|
||||
sdkSource: forge("mesh-sdk"),
|
||||
catalogSource: forge("mesh-catalog"),
|
||||
// A refusal is the answer, and a re-run is the second half of this bed, not a retry.
|
||||
attempts: 1,
|
||||
verify: false,
|
||||
log: (m) => console.log(m),
|
||||
});
|
||||
|
||||
// 1. From bare to the broker step, and no further: the catalogue the installer reads has no nats module.
|
||||
const withoutTheBus = mkdtempSync(join(tmpdir(), "mesh-lab-590-catalogue-"));
|
||||
try {
|
||||
const nats = join(catalogDir, "nats");
|
||||
cpSync(catalogDir, withoutTheBus, { recursive: true, filter: (src) => src !== nats && !src.startsWith(nats + "/") });
|
||||
first = await run(withoutTheBus);
|
||||
} catch (err) {
|
||||
unreached = `the installer could not be run: ${(err as Error).message}`;
|
||||
return;
|
||||
} finally {
|
||||
rmSync(withoutTheBus, { recursive: true, force: true });
|
||||
}
|
||||
report.push(...first.report);
|
||||
if (first.ok || stoppedAt(first) !== BROKER) {
|
||||
unreached = first.ok
|
||||
? "the first run finished without the nats module, so the broker step never stopped it"
|
||||
: `the first run stopped at ${first.step || "no step it named"}, before the broker step:\n${first.why}`;
|
||||
return;
|
||||
}
|
||||
if (!/nats module's manifest could not be read/.test(first.said)) {
|
||||
unreached = `the first run stopped at the broker step for another reason than the missing module:\n${first.why}`;
|
||||
return;
|
||||
}
|
||||
seenBefore = await look();
|
||||
report.push(` before the step ${busContainers(seenBefore.running).map((b) => `${b.name} on ${b.image}`).join("; ")}`);
|
||||
report.push(` ${written(seenBefore).length} stream(s) holding something: ` +
|
||||
written(seenBefore).map((s) => `${s.name} (last ${s.lastSeq})`).join(", "));
|
||||
if (written(seenBefore).length === 0) {
|
||||
unreached = "nothing was written on the bus before the broker step, so whether its streams survive it says nothing";
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. The whole catalogue: the steps before are done already, and the broker step adopts the bus.
|
||||
try {
|
||||
second = await run(catalogDir);
|
||||
} catch (err) {
|
||||
unreached = `the installer could not be run again: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
report.push(...second.report);
|
||||
if (!second.ok && BEFORE_BROKER.includes(stoppedAt(second))) {
|
||||
unreached = `the second run stopped at ${second.step}, before the broker step it was run again for:\n${second.why}`;
|
||||
return;
|
||||
}
|
||||
seenAfter = await look();
|
||||
report.push(` after the step ${busContainers(seenAfter.running).map((b) => `${b.name} on ${b.image}`).join("; ") || "no bus"}`);
|
||||
const nodes = await on(ANCHOR, `docker exec mesh-controller /mesh-controller node list`, 60_000);
|
||||
nodeLine = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${ANCHOR} `)) ?? "";
|
||||
report.push(` node list ${nodeLine || nodes.out.trim()}`);
|
||||
}, { timeout: 7_200_000 });
|
||||
|
||||
after(async () => {
|
||||
if (KEEP) {
|
||||
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
||||
return;
|
||||
}
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
test("genesis adopts its bus as the nats module in place: one bus, its streams kept, the node connected", {
|
||||
skip, timeout: 7_200_000,
|
||||
}, async (t) => {
|
||||
console.log(report.join("\n"));
|
||||
if (unreached) {
|
||||
t.skip(`ADR 0333 §5 NOT REACHED: ${unreached}`);
|
||||
return;
|
||||
}
|
||||
const ran = second as GenesisResult;
|
||||
// Stopped AT the broker step: the adoption itself refused or failed — the fault, not a missed bed.
|
||||
assert.ok(ran.ok || stoppedAt(ran) !== BROKER,
|
||||
`genesis stopped at its broker step and did not adopt the bus:\n${ran.why}`);
|
||||
if (!ran.ok) {
|
||||
console.log(`\nthe second run crossed the broker step and stopped later, at ${ran.step} — not this bed's ` +
|
||||
`question:\n${ran.why}`);
|
||||
}
|
||||
const wrong = busNotKept(seenBefore as Seen, seenAfter as Seen, nodeLine, ran.said);
|
||||
assert.deepEqual(wrong, [], `genesis did not keep its bus through the broker step:\n - ${wrong.join("\n - ")}`);
|
||||
console.log(`\nVERDICT: one bus, ${BUS}, on the module's image; ${written(seenBefore as Seen).length} stream(s) ` +
|
||||
`kept; ${ANCHOR} connected.`);
|
||||
});
|
||||
@@ -12,7 +12,7 @@
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||||
* MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
* MESH_LAB_KEEP=1 to leave it standing afterwards
|
||||
*/
|
||||
|
||||
@@ -112,7 +112,7 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
||||
const registry = o.registry ?? "127.0.0.1:5000";
|
||||
// The registry's and the builder's. Not the control plane's: its manifest lives in its own
|
||||
// repository and genesis takes it from the build it runs (novox/hq ADR 0069, 04-ISSUES/072).
|
||||
const modules = o.catalogueModules ?? ["distribution", "builder"];
|
||||
const modules = o.catalogueModules ?? ["distribution", "build-agent"];
|
||||
const catalogueOnMachine = o.catalogueOnMachine ?? "/opt/mesh-catalog";
|
||||
const log = o.log ?? (() => {});
|
||||
|
||||
|
||||
+13
-12
@@ -23,14 +23,14 @@ import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImag
|
||||
/**
|
||||
* The example bundle in mesh-host names a registry that no longer exists.
|
||||
*
|
||||
* `examples/foundation-first-node-nats.lock` was written **for a target**, and the target was the lab: it
|
||||
* `examples/foundation-first-node.lock` was written **for a target**, and the target was the lab: it
|
||||
* pins `192.0.2.250:5000/…` because that is where the registry the lab used to raise served from.
|
||||
* That registry is gone, so those three references name nothing.
|
||||
*
|
||||
* Two of them are ordinary third-party images and belong to the internet. Rather than invent
|
||||
* digests here, they are the ones the mesh's own modules already pin — mesh-catalog's `postgres`
|
||||
* and `lavinmq` — so the foundation's store and broker are literally the images the mesh runs. The
|
||||
* third, mesh-controller, exists in no registry at all and becomes the ID the machine holds it under.
|
||||
* Two of them are ordinary third-party images and belong to the internet. The store's is the one
|
||||
* mesh-catalog's `postgres` module already pins, so the foundation's store is literally the image the
|
||||
* mesh runs; the bus's is below. The third, mesh-controller, exists in no registry at all and becomes
|
||||
* the ID the machine holds it under.
|
||||
*
|
||||
* **The bundle itself should be fixed in mesh-host**, and this substitution deleted with it. It is
|
||||
* here because the file lives in another repository and because a fixture that lies about where an
|
||||
@@ -38,11 +38,9 @@ import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImag
|
||||
*/
|
||||
const UPSTREAM_STORE =
|
||||
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
|
||||
const UPSTREAM_BROKER =
|
||||
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
|
||||
/**
|
||||
* And the bus, for `foundation-first-node-nats.lock` — the bundle the mesh raises since it stopped
|
||||
* speaking AMQP (novox/hq ADR 0131). The digest is the bundle's own: what the registry served was a
|
||||
* And the bus, for `foundation-first-node.lock` — the one genesis bundle, which raises the bus the
|
||||
* mesh runs on since it stopped speaking AMQP (novox/hq ADR 0131, ADR 0331). The digest is the bundle's own: what the registry served was a
|
||||
* copy of the upstream image, so the same digest resolves on Docker Hub, and this is a prefix being
|
||||
* removed rather than a reference being replaced.
|
||||
*/
|
||||
@@ -59,8 +57,6 @@ const UPSTREAM_BUS =
|
||||
export function foundationBundle(path: string, held: HeldImage[]): string {
|
||||
let text = readFileSync(path, "utf8");
|
||||
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
|
||||
text = text.replaceAll(
|
||||
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
|
||||
text = text.replaceAll(/[A-Za-z0-9_.:-]+:[0-9]+\/nats@sha256:[0-9a-f]{64}/g, UPSTREAM_BUS);
|
||||
return pinnedInto(text, held);
|
||||
}
|
||||
@@ -376,11 +372,16 @@ function shellQuote(s: string): string {
|
||||
*/
|
||||
export async function deriveTheFilterOn(o: {
|
||||
machine: string; node: string; hubPort: number;
|
||||
/** The images the machines hold. Given, the manifest is the lab's form of the catalogue's
|
||||
* (catalogueModule): its build section gone, since the lab builds nothing. The filter's verbs are a
|
||||
* tools bundle the node's runtime launches (novox/hq ADR 0170, 0193), so no container of the
|
||||
* module names an image and none need be stocked; what a bed needs of it is the ruleset it loads. */
|
||||
held?: HeldImage[];
|
||||
must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
|
||||
mesh: (command: string, timeoutMs?: number) => Promise<string>;
|
||||
on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>;
|
||||
}): Promise<string> {
|
||||
const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
|
||||
const manifest = o.held ? catalogueModule(FILTER_MODULE, o.held) : readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
|
||||
await o.must(o.machine,
|
||||
`printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`);
|
||||
await o.mesh(`module add /${FILTER_MODULE}.json`);
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* **A machine joins through the tunnel, and the bus is never public** (novox/hq ADR 0169).
|
||||
*
|
||||
* The anchor raises the foundation from the bundle, joins over its own loopback — it runs the bus,
|
||||
* so it needs no tunnel to reach it — and is placed as the hub. The joiner then does what a new
|
||||
* machine does: makes its tunnel key and prints the public half, is issued a token for that key,
|
||||
* and enrols with the token alone. The bus is closed to the joiner's own address on the anchor
|
||||
* before any of that, so the enrolment can arrive only over the tunnel the token gave it.
|
||||
*
|
||||
* It asserts the ADR's last row: *a new machine joins from outside the hub's network with the bus
|
||||
* closed to it.* The rows before it are the controller's own tests.
|
||||
*
|
||||
* It needs a host binary, the foundation bundle and the control plane's image:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* (mesh-controller:development, from mesh-controller's `make image`)
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
|
||||
const skip = !capability.usable
|
||||
? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary)
|
||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle)
|
||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||
: false;
|
||||
|
||||
const SCENARIO = "joins-through-the-tunnel";
|
||||
/** The hub's tunnel port, and the bus's port as the bundle publishes it on the anchor. */
|
||||
const HUB_PORT = 51820;
|
||||
const BUS_PORT = 5671;
|
||||
let instanceId = "";
|
||||
|
||||
function quote(s: string): string {
|
||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, [
|
||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||
], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||
}
|
||||
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
assert.ok(found, `no token in:\n${said}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
|
||||
const end = Date.now() + within;
|
||||
while (Date.now() < end) {
|
||||
if (await check()) return;
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
|
||||
*
|
||||
* **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146):
|
||||
* the bus here is the bundle's, so an account the mesh composes reaches it only if whoever raised it
|
||||
* places it — the enrolment's when a token is issued, the node's own once it enrols.
|
||||
*/
|
||||
async function placeTheBusUsers(): Promise<void> {
|
||||
await must("anchor",
|
||||
`docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
|
||||
`docker kill -s HUP mesh-broker >/dev/null`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
|
||||
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 900_000);
|
||||
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||
}
|
||||
}, { timeout: 1_800_000 });
|
||||
|
||||
after(async () => {
|
||||
if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 600_000 });
|
||||
|
||||
test("the machine that runs the bus joins over its own loopback and becomes the hub", { skip, timeout: 900_000 }, async () => {
|
||||
await mesh("node add anchor");
|
||||
const token = tokenFrom(await mesh("token issue --node anchor"));
|
||||
await placeTheBusUsers();
|
||||
const said = await must("anchor", `${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
assert.match(said, /enrolled as anchor/, said);
|
||||
assert.doesNotMatch(said, /tunnel to the hub/, `the machine running the bus went through a tunnel:\n${said}`);
|
||||
await placeTheBusUsers();
|
||||
await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
||||
|
||||
await mesh(`overlay place anchor --hub --endpoint 192.0.2.10:${HUB_PORT} --site lab`);
|
||||
await mesh("assign anchor mesh-wireguard");
|
||||
await mesh("push anchor", 600_000);
|
||||
await until("the hub's tunnel coming up", 300_000,
|
||||
async () => (await on("anchor", `wg show mesh0 listen-port`)).out.trim() === String(HUB_PORT),
|
||||
async () => `--- anchor host ---\n${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
||||
// The hub's port, opened as the derived filter opens it — scenery here: the filter module is the
|
||||
// two-node walk's to prove, and the bundle's own filter admits only ssh, the bus and the registry.
|
||||
await must("anchor", `nft insert rule inet mesh input udp dport ${HUB_PORT} accept`);
|
||||
});
|
||||
|
||||
test("a machine joins through the tunnel, with the bus closed to it", { skip, timeout: 900_000 }, async () => {
|
||||
// **The bus closed to the joiner**, at the anchor's very first hook — before the container
|
||||
// runtime's forwarding — so nothing from its own address reaches the bus, by any path.
|
||||
await must("anchor", `nft add table ip lab_bus_closed && ` +
|
||||
`nft add chain ip lab_bus_closed pre '{ type filter hook prerouting priority -300; }' && ` +
|
||||
`nft add rule ip lab_bus_closed pre ip saddr 192.0.2.30 tcp dport ${BUS_PORT} drop`);
|
||||
const reached = await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`);
|
||||
assert.ok(!reached.ok, "the bus is still reachable from the joiner's own address; the bed proves nothing");
|
||||
|
||||
await must("joiner", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`, 300_000);
|
||||
const key = (await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim();
|
||||
assert.match(key, /^[A-Za-z0-9+/]{43}=$/, `not a tunnel key: ${key}`);
|
||||
// Asked again, the same key: a token may already have been issued for it.
|
||||
assert.equal((await must("joiner", `${HOST_PATH} key 2>/dev/null`)).trim(), key);
|
||||
|
||||
// The token is issued for the key; the hub is sent the machine as a peer before it is shown.
|
||||
const issued = await mesh(`token issue --new joiner --overlay-key ${key}`, 600_000);
|
||||
const token = tokenFrom(issued);
|
||||
const carried = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as {
|
||||
broker: string; tunnel?: { key: string; address: string; hub_endpoint: string };
|
||||
};
|
||||
assert.equal(carried.tunnel?.key, key, `the token was not issued for the machine's key: ${JSON.stringify(carried)}`);
|
||||
assert.equal(carried.tunnel?.hub_endpoint, `192.0.2.10:${HUB_PORT}`);
|
||||
assert.doesNotMatch(carried.broker, /^192\.0\.2\./, `the token sends the machine to the bus's public address: ${carried.broker}`);
|
||||
await placeTheBusUsers();
|
||||
|
||||
const said = await must("joiner", `${HOST_PATH} enrol --token ${quote(token)}`, 300_000);
|
||||
assert.match(said, /the tunnel to the hub is up/, said);
|
||||
assert.match(said, /the hub answered the tunnel/, said);
|
||||
assert.match(said, /enrolled as joiner/, said);
|
||||
|
||||
const shakes = await must("joiner", `wg show mesh0 latest-handshakes`);
|
||||
assert.ok(/\s[1-9]\d*\s*$/m.test(shakes), `the tunnel never shook hands with the hub: ${shakes}`);
|
||||
// And the bus is still closed to it: it joined through the tunnel, not around it.
|
||||
assert.ok(!(await on("joiner", `timeout 5 bash -c '</dev/tcp/192.0.2.10/${BUS_PORT}'`)).ok,
|
||||
"the bus opened to the joiner's own address during the join");
|
||||
|
||||
const recorded = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||
`-c "select overlay_key from node where name = 'joiner'"`)).trim();
|
||||
assert.equal(recorded, key, "the mesh does not hold the tunnel key the machine made");
|
||||
});
|
||||
@@ -25,7 +25,7 @@
|
||||
* its vhost — the provider named the vhost after the login — and nothing is hardcoded; the provider's
|
||||
* `serves` carries the port so the consumer references `${bound:amqp:port}`.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*
|
||||
* HELPER — stock the two runtimes into the local daemon before the run (some may already be there):
|
||||
* scripts/build-module-runtime.sh lavinmq /tmp/lavinmq.tar
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
* asserts the templated URL and that a request to it reaches the running server (ollama answers
|
||||
* /v1/models even with no model pulled — the wiring is what is proven, not a model's output).
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* No module runtime image is built — both modules are pure declaration.
|
||||
*/
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
* Mint on one side and create on the other agreeing, with no shared key and nothing placed by the
|
||||
* test, is the entire provider/consumer contract working as one thing.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development, which
|
||||
* scenarios/redis-node.yml stocks.
|
||||
*/
|
||||
|
||||
+111
-23
@@ -9,7 +9,7 @@
|
||||
* It needs a host binary and the foundation bundle:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*
|
||||
* The bundle's image references are rewritten to the ones this scenario's own registry serves.
|
||||
* A digest belongs to whatever registry serves it, so a committed bundle names a registry that is
|
||||
@@ -193,6 +193,20 @@ function bundleFor(images: HeldImage[]): string {
|
||||
return foundationBundle(bundle, images);
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the mesh's composed user list where the anchor's bus reads it, and make it re-read.
|
||||
*
|
||||
* **Genesis's step, done by hand because this bed raises genesis by hand** (novox/hq 04-ISSUES/146).
|
||||
* The bus here is the bundle's, not a module the mesh delivers, so an account the mesh composes —
|
||||
* the enrolment when a token is issued, the node's own when it enrols — reaches it only if whoever
|
||||
* raised it places it. Without this the first join is refused with an authorisation violation.
|
||||
*/
|
||||
async function composeTheBusUsers(): Promise<void> {
|
||||
await must("anchor",
|
||||
`docker exec mesh-controller /mesh-controller broker accounts > /var/lib/mesh-bus-conf/accounts.conf && ` +
|
||||
`docker kill -s HUP mesh-broker >/dev/null`);
|
||||
}
|
||||
|
||||
/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
|
||||
function tokenFrom(said: string): string {
|
||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||
@@ -200,8 +214,33 @@ function tokenFrom(said: string): string {
|
||||
return found;
|
||||
}
|
||||
|
||||
/** The builder started by hand on the anchor, against the foundation broker's plain port on
|
||||
* loopback — the one that builds until a builder module can (see the retired test's note). */
|
||||
/**
|
||||
* The build machine's module: a holder of the node-build-agent seat, with the bus credential the mesh
|
||||
* seals to the machine and nothing else.
|
||||
*
|
||||
* **On the bus, the way today's build agent is** (novox/hq ADR 0106, 0127, 0131). The builder used to
|
||||
* be started against the old broker's guest account on loopback; the mesh moved to NATS, the builder
|
||||
* dials only a credential the mesh delivered (`MESH_BROKER_FILE`), and that account exists only as a
|
||||
* module's, composed into the bus's user list. The catalogue's `build-agent` is that module, but it
|
||||
* runs the builder as a container from an artifact the mesh builds and needs the artifact store and
|
||||
* package registry bound; this bed builds nothing before it has a builder, so it declares the same
|
||||
* claim and the same sealed credential and starts the binary it was pointed at itself.
|
||||
*/
|
||||
const BUILDER_MODULE = "lab-builder";
|
||||
const BUILDER_STATE = `/var/lib/mesh/${BUILDER_MODULE}`;
|
||||
const BUILDER_MANIFEST = JSON.stringify({
|
||||
module: BUILDER_MODULE, version: "1",
|
||||
claims: [{ name: "node-build-agent", scope: "node", serves: ["current", "kill", "pause", "resume"] }],
|
||||
"own-secrets": { broker: `${BUILDER_STATE}/broker` },
|
||||
resources: [{ id: "mesh-state", type: "directory", path: BUILDER_STATE, mode: "0700" }],
|
||||
});
|
||||
|
||||
/**
|
||||
* The builder, on the anchor, holding the build seat over the bus with the credential the mesh sealed
|
||||
* to the machine. Assigning the module issues the account (novox/hq issue 203); the bus here is the
|
||||
* bundle's, so its user list is placed by hand (composeTheBusUsers); the push writes the credential
|
||||
* where the module declared it. Idempotent: a warm restore or a second call starts only what is gone.
|
||||
*/
|
||||
async function startBuilder(): Promise<void> {
|
||||
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
|
||||
// return, so it is pushed whenever the machine has none.
|
||||
@@ -211,11 +250,31 @@ async function startBuilder(): Promise<void> {
|
||||
"--mode", "0755",
|
||||
], 180_000);
|
||||
}
|
||||
if (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
|
||||
await must("anchor", `printf %s ${quote(BUILDER_MANIFEST)} > /tmp/${BUILDER_MODULE}.json && ` +
|
||||
`docker cp /tmp/${BUILDER_MODULE}.json mesh-controller:/${BUILDER_MODULE}.json`);
|
||||
await mesh(`module add /${BUILDER_MODULE}.json`);
|
||||
await mesh(`assign anchor ${BUILDER_MODULE}`);
|
||||
await composeTheBusUsers();
|
||||
await mesh("push anchor", 600_000);
|
||||
const by = Date.now() + 180_000;
|
||||
while (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
|
||||
if (Date.now() > by) {
|
||||
assert.fail(`the mesh never delivered the builder's bus credential to ${BUILDER_STATE}/broker:\n` +
|
||||
(await on("anchor", "tail -30 /var/log/mesh-host.log")).out);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
}
|
||||
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
||||
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
|
||||
`(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
||||
`(MESH_BROKER_FILE=${BUILDER_STATE}/broker MESH_NODE=anchor MESH_REGISTRY=${registry} ` +
|
||||
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
||||
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
|
||||
// Holding the seat, not merely started: a builder refused by the bus exits, and a build asked of a
|
||||
// seat nobody holds waits out its whole timeout before saying anything.
|
||||
const running = await on("anchor", `pgrep -x mesh-builder >/dev/null`);
|
||||
assert.ok(running.ok, `the builder did not stay up:\n${(await on("anchor", "tail -30 /var/log/mesh-builder.log")).out}`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
@@ -250,8 +309,7 @@ before(async () => {
|
||||
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
|
||||
assert.equal(running.out.trim(), "yes",
|
||||
"the host did not come back after a restore, so nothing would apply anything");
|
||||
// The hand-started builder is memory too, and the snapshot is disk.
|
||||
if (builder) await startBuilder();
|
||||
// The hand-started builder is memory too; the build test starts it again where it is needed.
|
||||
|
||||
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
|
||||
`and started the host again`);
|
||||
@@ -276,9 +334,8 @@ before(async () => {
|
||||
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`);
|
||||
|
||||
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
||||
// assumed: nothing else in this scenario would start one.
|
||||
if (builder) await startBuilder();
|
||||
// No build machine yet: the builder holds its seat on a credential the mesh delivers, so it is
|
||||
// started by the test that builds, once the anchor has joined (startBuilder).
|
||||
if (warming) {
|
||||
// Snapshotted only now, with everything up: a state worth returning to is the one after the
|
||||
// part nobody wants to repeat.
|
||||
@@ -308,15 +365,44 @@ test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
|
||||
});
|
||||
|
||||
test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
|
||||
for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
|
||||
await mesh(`node add ${node}`);
|
||||
const token = tokenFrom(await mesh(`token issue --node ${node}`));
|
||||
// No --name. The token says what the mesh calls the machine, which is the fault this walk
|
||||
// found the first time it was run.
|
||||
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
||||
assert.match(said, new RegExp(`enrolled as ${node}`), said);
|
||||
assert.match(said, /sealing key/, "no sealing key was generated");
|
||||
// **The anchor runs the bus, so it joins over its own loopback.** Every other machine joins
|
||||
// through the tunnel (novox/hq ADR 0169): the anchor is the hub, and its tunnel comes up first.
|
||||
await mesh(`node add anchor`);
|
||||
const first = tokenFrom(await mesh(`token issue --node anchor`));
|
||||
await composeTheBusUsers();
|
||||
// No --name. The token says what the mesh calls the machine, which is the fault this walk
|
||||
// found the first time it was run.
|
||||
const anchorSaid = await must("anchor", `${HOST_PATH} enrol --token ${quote(first)}`);
|
||||
await composeTheBusUsers();
|
||||
assert.match(anchorSaid, /enrolled as anchor/, anchorSaid);
|
||||
assert.match(anchorSaid, /sealing key/, "no sealing key was generated");
|
||||
|
||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||
await mesh("assign anchor networking");
|
||||
await must("anchor", `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
||||
await mesh("push anchor");
|
||||
// The host reports which of its links face outside with every apply, and the filter is not
|
||||
// composed for a machine that has not — so its first apply is waited for, not slept past.
|
||||
const appliedBy = Date.now() + 180_000;
|
||||
while (!(await on("anchor", `grep -q 'applied [0-9]' /var/log/mesh-host.log`)).ok) {
|
||||
if (Date.now() > appliedBy) assert.fail(`the anchor never applied what it was sent:\n${(await on("anchor", "tail -30 /var/log/mesh-host.log")).out}`);
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on });
|
||||
|
||||
// **The laptop makes its tunnel key, and the token is issued for it.** The hub is told the key
|
||||
// before the token is shown, so the tunnel answers the first time the laptop knocks.
|
||||
await must("laptop", `pacman -Sy --noconfirm --needed wireguard-tools >/dev/null 2>&1`);
|
||||
const key = (await must("laptop", `${HOST_PATH} key 2>/dev/null`)).trim();
|
||||
await mesh(`node add laptop`);
|
||||
const second = tokenFrom(await mesh(`token issue --node laptop --overlay-key ${key}`));
|
||||
await composeTheBusUsers();
|
||||
const laptopSaid = await must("laptop", `${HOST_PATH} enrol --token ${quote(second)}`);
|
||||
await composeTheBusUsers();
|
||||
assert.match(laptopSaid, /the tunnel to the hub is up/, laptopSaid);
|
||||
assert.match(laptopSaid, /enrolled as laptop/, laptopSaid);
|
||||
assert.match(laptopSaid, /sealing key/, "no sealing key was generated");
|
||||
|
||||
const recorded = await must("anchor",
|
||||
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||
@@ -366,7 +452,7 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
|
||||
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
||||
// and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
|
||||
// keeps the hub closed and nothing on the laptop reaches anchor over the private network.
|
||||
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
||||
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, held, must, mesh, on });
|
||||
|
||||
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
|
||||
// Named after the machine *and* the module, because a consumer is both (novox/hq
|
||||
@@ -508,11 +594,11 @@ test("a declaration waits for a machine that is switched off", { skip, timeout:
|
||||
// Everything needed to tell "the message was never queued" from "the host never read it".
|
||||
const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
|
||||
const queues = await on("anchor",
|
||||
`docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
|
||||
`curl -s 'http://127.0.0.1:8222/jsz?consumers=true' 2>&1 | head -c 4000`);
|
||||
const owned = await on("anchor",
|
||||
`docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
|
||||
assert.fail(`a declaration sent to a switched-off machine was lost\n` +
|
||||
`--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
|
||||
`--- the host's log ---\n${log.out}\n--- the bus's streams and consumers ---\n${queues.out}\n` +
|
||||
`--- what each machine last did ---\n${owned.out}`);
|
||||
}
|
||||
assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
|
||||
@@ -1156,9 +1242,8 @@ test("a new commit reaches a machine that is already running the old one", {
|
||||
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
|
||||
// question "did my change go out?". This is that question, end to end — a commit, a build, a
|
||||
// catalogue, and a machine that ends up running what the source says.
|
||||
// The hand-started builder does not outlive a broker restart, and the foundation's broker is
|
||||
// recreated when the first push reconciles it: a builder is (re)started here, where a build is
|
||||
// asked for. The mesh's own builder is a module with a restart policy and needs none of this.
|
||||
// The builder needs the anchor joined — its credential is the mesh's to deliver — so it is started
|
||||
// here, where a build is asked for. The mesh's own build agent is a module with a restart policy.
|
||||
await startBuilder();
|
||||
const repo = "/var/lib/mesh/builder/repositories/delivered";
|
||||
const write = async (what: string) =>
|
||||
@@ -1662,3 +1747,6 @@ test("a third-party workload is adopted, with the credential it already had", {
|
||||
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
|
||||
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
|
||||
// mesh-grant-end-to-end, against the catalogue's redis.
|
||||
|
||||
// A machine joining through the tunnel with the bus closed to it is its own bed:
|
||||
// joins-through-the-tunnel.test.ts (novox/hq ADR 0169).
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||||
* MESH_LAB_BOOTSTRAP_BINARY=.../mesh-host/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
* MESH_LAB_SOURCE=<forge>/mesh-controller.git MESH_LAB_SOURCE_REF=<commit>
|
||||
* MESH_LAB_KEEP=1 to leave it standing afterwards
|
||||
@@ -109,7 +109,7 @@ const CONTROL_PLANE = { module: "mesh-controller", repo: "mesh-controller", path
|
||||
* the registry and the builder — are here too: they are carried in, and a mesh missing any of them
|
||||
* is not one.
|
||||
*/
|
||||
const MUST_HOLD = ["mesh-controller", "distribution", "builder", "mesh-tools", "postgres",
|
||||
const MUST_HOLD = ["mesh-controller", "distribution", "build-agent", "mesh-tools", "postgres",
|
||||
"mesh-catalog", "lavinmq", "mesh-vault", "amqp-ping"];
|
||||
const MUST_RUN = ["mesh-controller", "mesh-registry", "mesh-broker", "mesh-store",
|
||||
"mesh-postgres", "mesh-catalog", "mesh-lavinmq", "mesh-vault", "amqp-ping"];
|
||||
@@ -898,7 +898,7 @@ before(async () => {
|
||||
// does. novox/hq issue 050 — on a fresh mesh the modules built before the catalogue existed
|
||||
// are exactly the ones it needed in order to exist, so the hole is always the foundation.
|
||||
const missing = MUST_HOLD.filter((m) =>
|
||||
!["distribution", "builder"].includes(m) && !held.includes(m));
|
||||
!["distribution", "build-agent"].includes(m) && !held.includes(m));
|
||||
assert.deepEqual(missing, [],
|
||||
`the catalogue does not hold ${missing.join(", ")} — the mesh built them and its own ` +
|
||||
`record has no trace of it (novox/hq issue 050):\n${held}`);
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
* ordinary sealed-delivery path — with NO manager, NO refresh, NO access/refresh split, NO usage. The
|
||||
* whole of OpenAI's integration in the control plane is one registry line (vendor -> static-key).
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* Build the consumer runtime image into the local daemon first:
|
||||
* scripts/build-module-runtime.sh openai-consumer /tmp/openai-consumer.tar
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* REPLAY R92 — genesis pushes to its own registry on a runtime nobody told to trust it (novox/hq issue 92).
|
||||
*
|
||||
* On the first real genesis the push of the control plane's image into the registry genesis had just
|
||||
* raised was refused: `Get "https://<address>/v2/": http: server gave HTTP response to HTTPS client`. The
|
||||
* registry speaks plain HTTP on purpose (ADR 0082), and the trust that lets a runtime speak to it was
|
||||
* written by the `docker` module, installed six steps after the push. The fix (mesh-host #91) has genesis
|
||||
* write that trust into the runtime's configuration itself, and confirm it through `docker info`, before
|
||||
* anything is pushed.
|
||||
*
|
||||
* **The lab hid it.** Its base image trusts the documentation ranges as plain-HTTP registries before any bed
|
||||
* starts (src/lifecycle/base.ts), so every genesis bed pushed to a registry its runtime already trusted. This
|
||||
* bed takes that trust back off the anchor — the base's file removed, the runtime restarted, and the runtime
|
||||
* asked that it now trusts loopback and nothing else — and then runs the installer as an operator does, with
|
||||
* `--registry 192.0.2.10:5000`.
|
||||
*
|
||||
* What it holds: genesis gets past its publish step, the registry it raised serves the control plane's image
|
||||
* it pushed by the address it was given, and the runtime trusts that registry by the mesh's name too, which
|
||||
* the builder pushes to later. Before the fix the push is refused, and the bed fails naming it.
|
||||
*
|
||||
* **A bed that could not reach the step it replays says so and does not fail**: a genesis stopped before
|
||||
* its registry step — a pull the internet refused, a build that broke — proves nothing about the order this
|
||||
* replays, and the test is skipped with the reason, which the prover reads as an error, never as the fault
|
||||
* reproduced. Genesis is run once and not asked to finish: the steps after the control plane read
|
||||
* repositories this bed does not serve, and are not what it replays.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock (the installer refuses the bus one)
|
||||
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_REPLAY_CONTROLLER=<a mesh-controller checkout>
|
||||
* MESH_LAB_KEEP=1 leave it standing
|
||||
*
|
||||
* Run by `go run ./replays/cmd/prove R92`, which builds all of that at the fix and at the commit before it.
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||||
import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||
import { genesis, type GenesisResult } from "./genesis.ts";
|
||||
import { carry } from "./forge.ts";
|
||||
|
||||
const SCENARIO = "genesis-untrusted";
|
||||
const ANCHOR = "anchor";
|
||||
const FORGE = "forge";
|
||||
const ANCHOR_ADDRESS = "192.0.2.10";
|
||||
const FORGE_ADDRESS = "192.0.2.30";
|
||||
/** The registry as the operator gives it: the anchor's own address and the registry's port. */
|
||||
const REGISTRY = `${ANCHOR_ADDRESS}:5000`;
|
||||
/** And as the mesh names it: the node, the mesh's internal suffix, the same port. */
|
||||
const REGISTRY_BY_NAME = `${ANCHOR}.internal:5000`;
|
||||
/** The installer's steps up to and including the one this replays, in its order. */
|
||||
const UP_TO_PUBLISH = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry", "publish"];
|
||||
/** The steps before the registry's: a stop there never reached what this replays. */
|
||||
const BEFORE_REGISTRY = UP_TO_PUBLISH.slice(0, UP_TO_PUBLISH.indexOf("registry"));
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const installer = bootstrapBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||
const controller = process.env["MESH_REPLAY_CONTROLLER"] ?? "";
|
||||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-untrusted-live" : undefined);
|
||||
|
||||
const skip =
|
||||
!capability.usable ? capability.why :
|
||||
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
||||
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||
!controller || !existsSync(resolve(controller, ".git")) ? "MESH_REPLAY_CONTROLLER is not set to a mesh-controller checkout" :
|
||||
false;
|
||||
|
||||
let instanceId = "";
|
||||
/** Why the bed never reached the step it replays, or "" when it did. */
|
||||
let unreached = "";
|
||||
let result: GenesisResult | undefined;
|
||||
let untrusted = "";
|
||||
const report: string[] = [];
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The control plane's repository at the checkout's HEAD, served from the forge machine over git's protocol. */
|
||||
async function raiseForge(): Promise<string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-r92-forge-"));
|
||||
let head: string;
|
||||
try {
|
||||
head = carry(controller, join(dir, "mesh-controller.git"));
|
||||
const tar = join(tmpdir(), `mesh-lab-r92-forge-${process.pid}.tar`);
|
||||
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||
await push(instanceId, FORGE, tar, "/tmp/forge.tar");
|
||||
rmSync(tar, { force: true });
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
// Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing.
|
||||
await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`);
|
||||
for (let i = 0; ; i++) {
|
||||
if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-controller.git main`)).ok) break;
|
||||
if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`);
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
return head;
|
||||
}
|
||||
|
||||
/**
|
||||
* The anchor's runtime made a bare machine's: the base's trust of the documentation ranges taken off, and
|
||||
* the runtime asked that it trusts no registry but loopback. Before genesis, so nothing runs that a restart
|
||||
* would stop. Answers what the runtime said.
|
||||
*/
|
||||
async function untrust(): Promise<string> {
|
||||
await must(ANCHOR, `rm -f /etc/docker/daemon.json && systemctl restart docker`, 180_000);
|
||||
for (let i = 0; ; i++) {
|
||||
if ((await on(ANCHOR, `docker info >/dev/null`)).ok) break;
|
||||
if (i >= 60) throw new Error("the anchor's runtime did not answer after its trust was taken off");
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig}}'`)).trim();
|
||||
const parsed = JSON.parse(config) as { InsecureRegistryCIDRs?: string[]; IndexConfigs?: Record<string, { Secure?: boolean }> };
|
||||
const cidrs = parsed.InsecureRegistryCIDRs ?? [];
|
||||
const insecureNames = Object.entries(parsed.IndexConfigs ?? {}).filter(([, c]) => c.Secure === false).map(([n]) => n);
|
||||
if (cidrs.some((c) => c !== "127.0.0.0/8" && c !== "::1/128") || insecureNames.length > 0) {
|
||||
throw new Error(`the anchor's runtime still trusts more than loopback after the base's trust was taken off: ${config}`);
|
||||
}
|
||||
return `trusts ${cidrs.join(", ") || "nothing"} as plain-HTTP registries, and no registry by name`;
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||
onProgress: (m) => console.log(`raise: ${m}`),
|
||||
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||
});
|
||||
instanceId = raised.instanceId;
|
||||
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||
|
||||
let head = "";
|
||||
try {
|
||||
head = await raiseForge();
|
||||
report.push(` forge git://${FORGE_ADDRESS}/mesh-controller.git at ${head.slice(0, 8)}`);
|
||||
untrusted = await untrust();
|
||||
report.push(` anchor's runtime ${untrusted}`);
|
||||
} catch (err) {
|
||||
unreached = `the bed was not prepared, so the installer never ran: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`;
|
||||
try {
|
||||
result = await genesis({
|
||||
instanceId,
|
||||
node: ANCHOR,
|
||||
installer: installer as string,
|
||||
catalogDir,
|
||||
// The registry's manifest is all the steps this replays read from the catalogue.
|
||||
catalogueModules: ["distribution"],
|
||||
bundleTemplate: foundationBundle(bundle, []),
|
||||
registry: REGISTRY,
|
||||
source: forge("mesh-controller"),
|
||||
sourceRef: head,
|
||||
// Named because the installer requires them; served by nobody, since the steps that clone them
|
||||
// come after the one replayed here.
|
||||
toolsSource: forge("mesh-tools"),
|
||||
sdkSource: forge("mesh-sdk"),
|
||||
catalogSource: forge("mesh-catalog"),
|
||||
attempts: 1,
|
||||
verify: false,
|
||||
log: (m) => console.log(m),
|
||||
});
|
||||
} catch (err) {
|
||||
unreached = `the installer could not be run: ${(err as Error).message}`;
|
||||
return;
|
||||
}
|
||||
report.push(...result.report);
|
||||
const stoppedAt = result.step.replace(/^step \d+ of \d+, /, "");
|
||||
if (!result.ok && (BEFORE_REGISTRY.includes(stoppedAt) || !result.step)) {
|
||||
unreached = `genesis stopped at ${result.step || "no step it named"}, before its registry step, so the order ` +
|
||||
`this replays was never reached:\n${result.why}`;
|
||||
}
|
||||
}, { timeout: 3_600_000 });
|
||||
|
||||
after(async () => {
|
||||
if (KEEP) {
|
||||
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
||||
return;
|
||||
}
|
||||
if (instanceId) await destroy(instanceId);
|
||||
await destroyAll(`${SCENARIO}-`);
|
||||
}, { timeout: 900_000 });
|
||||
|
||||
test("genesis pushes to the registry it raised on a runtime nobody told to trust it", {
|
||||
skip, timeout: 3_600_000,
|
||||
}, async (t) => {
|
||||
console.log(report.join("\n"));
|
||||
if (unreached) {
|
||||
t.skip(`R92 NOT REACHED: ${unreached}`);
|
||||
return;
|
||||
}
|
||||
const ran = result as GenesisResult;
|
||||
const stoppedAt = ran.step.replace(/^step \d+ of \d+, /, "");
|
||||
|
||||
// The incident's own words, and the step it stopped at.
|
||||
assert.doesNotMatch(ran.said, /server gave HTTP response to HTTPS client/,
|
||||
`the runtime refused to push to ${REGISTRY} over plain HTTP — the registry was not trusted before the push ` +
|
||||
`(stopped at ${ran.step || "no step"}):\n${ran.why}`);
|
||||
assert.ok(ran.ok || !UP_TO_PUBLISH.includes(stoppedAt),
|
||||
`genesis stopped at ${ran.step}, before its control plane was published to ${REGISTRY}:\n${ran.why}`);
|
||||
|
||||
// The registry agreeing: the image genesis pushed, by the address it was given.
|
||||
const tags = (await on(ANCHOR, `curl -s --max-time 10 http://${REGISTRY}/v2/mesh-controller/tags/list`)).out.trim();
|
||||
assert.match(tags, /"genesis"/, `${REGISTRY} does not serve the control plane genesis pushed: ${tags || "no answer"}`);
|
||||
|
||||
// And the runtime trusts the registry by the mesh's name as well: the builder pushes there later.
|
||||
const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig.IndexConfigs}}'`)).trim();
|
||||
const byName = (JSON.parse(config) as Record<string, { Secure?: boolean }>)[REGISTRY_BY_NAME];
|
||||
assert.ok(byName && byName.Secure === false,
|
||||
`the runtime does not trust ${REGISTRY_BY_NAME}, the name the mesh pushes to its registry by: ${config}`);
|
||||
});
|
||||
@@ -25,7 +25,7 @@
|
||||
* publicly-trusted certificate and answering an HTTP-01 challenge at the name — is proven separately
|
||||
* by certificates.test.ts, which drives the same proxy binary against a real ACME server (Pebble).
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-route-proxy-image.sh builds mesh-route-proxy:development into the local daemon;
|
||||
* scenarios/route-forwarding.yml stocks it and alpine:latest, and serves both by digest.
|
||||
*/
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
* the container was replaced (a new container id) and the config on disk carries the new value.
|
||||
* It builds the host from source (no --no-build), because the behaviour under test is the host's.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh grafana builds mesh-runtime-grafana:development, which
|
||||
* scenarios/grafana-node.yml stocks.
|
||||
*/
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
* It needs a host binary and the foundation bundle:
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
* references of OURS are rewritten to the IDs the machine holds, and the co-located
|
||||
* host-port collisions are remapped at load time (see REMAP).
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
*/
|
||||
|
||||
import { test, before, after } from "node:test";
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
* (whole-mesh-full-live) and NOT torn down — it is left standing and browsable. Without it the bed
|
||||
* behaves like every other: raise in before(), destroy in after().
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
*/
|
||||
|
||||
@@ -119,7 +119,7 @@ const CATALOGUE_ON_MACHINE = "/opt/mesh-catalog";
|
||||
* a third, this list is where the bed finds out, by the installer saying which manifest it could
|
||||
* not read.
|
||||
*/
|
||||
const CATALOGUE_MODULES = ["distribution", "builder"];
|
||||
const CATALOGUE_MODULES = ["distribution", "build-agent"];
|
||||
|
||||
/**
|
||||
* Where this mesh keeps its own images, as the anchor reaches it.
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
* host ports here (container ports unchanged); the provider ports the consumers actually connect to
|
||||
* (postgres 5432, minio 9000, mongodb 27017, mssql 1433) are left as-is. See REMAP below.
|
||||
*
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node-nats.lock
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||
* scripts/build-module-runtime.sh builds one runtime per module that has code; the route-proxy image
|
||||
* is built by scripts/build-route-proxy-image.sh; scenarios/whole-mesh-novox.yml stocks them all
|
||||
* alongside every server image.
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
// A run in which no test passed has proven nothing, and says so (novox/hq issue 693).
|
||||
//
|
||||
// Seen on the lab: the genesis bed stopped before its subject, skipped its only test as "NOT REACHED",
|
||||
// and the suite printed "0 passed, 0 failed" and exited 0, so the lab recorded the run as passed.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { judge, type Receipt, endToEnd } from "../src/lastrun.ts";
|
||||
import { judgeRun, oneLine, type Skip } from "../src/outcomes.ts";
|
||||
import { counted, exitOf, reportOn } from "../src/suite.ts";
|
||||
|
||||
// The suite as a person runs it: not as a child of this test runner, which would have it report to us.
|
||||
const suiteEnv = (state: string): NodeJS.ProcessEnv => {
|
||||
const env: NodeJS.ProcessEnv = { ...process.env, XDG_STATE_HOME: state, NO_COLOR: "1" };
|
||||
delete env["NODE_TEST_CONTEXT"];
|
||||
return env;
|
||||
};
|
||||
|
||||
const lab = fileURLToPath(new URL("..", import.meta.url));
|
||||
const unreached: Skip = {
|
||||
file: "test/integration/genesis-keeps-its-bus.test.ts",
|
||||
test: "genesis adopts its bus",
|
||||
reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:\n docker exited 1",
|
||||
};
|
||||
const noBed: Skip = { file: "test/integration/mesh.test.ts", test: "a builder builds", reason: "set MESH_LAB_BUILDER to a built mesh-builder" };
|
||||
|
||||
test("a run whose only test skipped did not pass", () => {
|
||||
const said = judgeRun({ passed: 0, failed: 0 }, [unreached]);
|
||||
assert.equal(said.passed, false);
|
||||
assert.match(said.why, /no test passed: 1 skipped, 1 of them as NOT REACHED/);
|
||||
});
|
||||
|
||||
test("a run that ran no test at all did not pass", () => {
|
||||
assert.deepEqual(judgeRun({ passed: 0, failed: 0 }, []), { passed: false, why: "no test passed: none ran" });
|
||||
});
|
||||
|
||||
test("a NOT REACHED skip beside tests that passed is still not passed", () => {
|
||||
const said = judgeRun({ passed: 3, failed: 0 }, [unreached]);
|
||||
assert.equal(said.passed, false);
|
||||
assert.match(said.why, /1 test\(s\) skipped as NOT REACHED/);
|
||||
});
|
||||
|
||||
// The design limit: a skip its file declares expected, beside tests that passed, leaves the run passed.
|
||||
test("an expected skip beside tests that passed leaves the run passed", () => {
|
||||
assert.deepEqual(judgeRun({ passed: 21, failed: 0 }, [noBed]), { passed: true, why: "" });
|
||||
});
|
||||
|
||||
test("the suite's exit is not 0 for a run that did not pass, and is the runner's otherwise", () => {
|
||||
assert.equal(exitOf(0, { passed: 0, failed: 0 }, [unreached]), 1);
|
||||
assert.equal(exitOf(0, { passed: 21, failed: 0 }, [noBed]), 0);
|
||||
assert.equal(exitOf(1, { passed: 20, failed: 1 }, []), 1);
|
||||
assert.equal(exitOf(0, { passed: null, failed: null }, []), 0, "unread totals are reportOn's case, not this one");
|
||||
});
|
||||
|
||||
test("the recorded line says what skipped and why, and NOT PASSED", () => {
|
||||
let kept: unknown;
|
||||
const said = reportOn({ passed: 0, failed: 0, skipped: 1 }, [unreached], (_p, _f, outcome) => {
|
||||
kept = outcome;
|
||||
return { against: { "mesh-lab": "7ed7da1" }, ran: [unreached.file] };
|
||||
});
|
||||
assert.match(said, /— 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1/);
|
||||
assert.match(said, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
|
||||
assert.match(said, /^skipped: genesis adopts its bus \(test\/integration\/genesis-keeps-its-bus\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step: docker exited 1$/m);
|
||||
assert.deepEqual(kept, {
|
||||
skipped: 1,
|
||||
skips: [{ ...unreached, reason: oneLine(unreached.reason) }],
|
||||
notPassed: "no test passed: 1 skipped, 1 of them as NOT REACHED",
|
||||
});
|
||||
});
|
||||
|
||||
// A test that prints a summary-shaped line before it skips must not stand for the runner's summary.
|
||||
test("the runner's last summary is read, not a line a test printed before it", () => {
|
||||
const output = "\u2139 pass 5\n\u2139 fail 0\n\u2139 skipped 0\n\ufe63 genesis adopts its bus (1ms) # NOT REACHED\n" +
|
||||
"\u2139 tests 1\n\u2139 pass 0\n\u2139 fail 0\n\u2139 skipped 1\n";
|
||||
const counts = counted(output);
|
||||
assert.deepEqual(counts, { passed: 0, failed: 0, skipped: 1 });
|
||||
assert.equal(exitOf(0, counts, [unreached]), 1);
|
||||
});
|
||||
|
||||
test("NOT REACHED is matched in any case", () => {
|
||||
const said = judgeRun({ passed: 2, failed: 0 }, [{ ...noBed, reason: "the broker step was not reached" }]);
|
||||
assert.equal(said.passed, false);
|
||||
});
|
||||
|
||||
test("the runner's skipped total is read", () => {
|
||||
assert.deepEqual(counted("ℹ tests 1\nℹ pass 0\nℹ fail 0\nℹ skipped 1\n"), { passed: 0, failed: 0, skipped: 1 });
|
||||
});
|
||||
|
||||
test("a receipt of a run that proved nothing is not current, and says why", () => {
|
||||
const receipt: Receipt = {
|
||||
at: "2026-10-14T00:33:11Z", passed: 0, failed: 0, against: { "mesh-lab": "aaa" }, ran: [endToEnd],
|
||||
skipped: 1, skips: [unreached], notPassed: "no test passed: 1 skipped",
|
||||
};
|
||||
const said = judge(receipt, new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
|
||||
assert.equal(said.current, false, said.lines.join("\n"));
|
||||
const text = said.lines.join("\n");
|
||||
assert.match(text, /did not pass: no test passed: 1 skipped/);
|
||||
assert.match(text, /skipped: genesis adopts its bus — ADR 0333 §5 NOT REACHED/);
|
||||
// An older receipt, from before the fields, with nothing passed, is judged by its count.
|
||||
const older = judge({ at: receipt.at, passed: 0, failed: 0, against: receipt.against, ran: [endToEnd] },
|
||||
new Date("2026-10-14T01:00:00Z"), { "mesh-lab": "aaa" });
|
||||
assert.equal(older.current, false);
|
||||
});
|
||||
|
||||
// The real runner, end to end: the suite command against a file whose only test skips as NOT REACHED,
|
||||
// the way the genesis bed does.
|
||||
test("the suite exits non-zero for a file whose only test skipped as NOT REACHED, and records why", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
|
||||
try {
|
||||
const file = join(dir, "unreached.test.ts");
|
||||
writeFileSync(file, [
|
||||
`import { test } from "node:test";`,
|
||||
`test("genesis adopts its bus", (t) => {`,
|
||||
` console.log("a line the bed printed");`,
|
||||
` t.skip("ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base:\\nNotFound: content digest");`,
|
||||
`});`,
|
||||
].join("\n"));
|
||||
const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
|
||||
cwd: lab, encoding: "utf8", env: suiteEnv(dir),
|
||||
});
|
||||
assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
|
||||
assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
|
||||
assert.match(ran.stdout, /^NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED$/m);
|
||||
assert.match(ran.stdout, /^skipped: genesis adopts its bus \(.*unreached\.test\.ts\) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest$/m);
|
||||
const receipt = JSON.parse(readFileSync(join(dir, "mesh-lab", "last-run.json"), "utf8")) as Receipt;
|
||||
assert.equal(receipt.skipped, 1);
|
||||
assert.equal(receipt.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
|
||||
assert.match(receipt.skips?.[0]?.reason ?? "", /NOT REACHED/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("the suite does not take a summary a test printed for the runner's", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
|
||||
try {
|
||||
const file = join(dir, "forged.test.ts");
|
||||
writeFileSync(file, [
|
||||
`import { test } from "node:test";`,
|
||||
`test("genesis adopts its bus", (t) => {`,
|
||||
` console.log("\\u2139 pass 5\\n\\u2139 fail 0");`,
|
||||
` t.skip("ADR 0333 §5 NOT REACHED: stopped at step 16");`,
|
||||
`});`,
|
||||
].join("\n"));
|
||||
const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
|
||||
cwd: lab, encoding: "utf8", env: suiteEnv(dir),
|
||||
});
|
||||
assert.equal(ran.status, 1, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
|
||||
assert.match(ran.stdout, /0 passed, 0 failed, 1 skipped/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("the suite still exits 0 for a file that passes a test and skips another for a declared reason", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-693-"));
|
||||
try {
|
||||
const file = join(dir, "declared.test.ts");
|
||||
writeFileSync(file, [
|
||||
`import { test } from "node:test";`,
|
||||
`test("a module lands", () => {});`,
|
||||
`test("a builder builds", { skip: "set MESH_LAB_BUILDER to a built mesh-builder" }, () => {});`,
|
||||
].join("\n"));
|
||||
const ran = spawnSync(process.execPath, ["--experimental-strip-types", join(lab, "src", "cli.ts"), "suite", "--no-build", file], {
|
||||
cwd: lab, encoding: "utf8", env: suiteEnv(dir),
|
||||
});
|
||||
assert.equal(ran.status, 0, `the suite exited ${ran.status}:\n${ran.stdout}${ran.stderr}`);
|
||||
assert.match(ran.stdout, /1 passed, 0 failed, 1 skipped/);
|
||||
assert.doesNotMatch(ran.stdout, /NOT PASSED/);
|
||||
assert.match(ran.stdout, /^skipped: a builder builds \(.*declared\.test\.ts\) # set MESH_LAB_BUILDER to a built mesh-builder$/m);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
+7
-6
@@ -5,7 +5,7 @@ import { counted, reportOn } from "../src/suite.ts";
|
||||
// The totals come from the runner's own summary, and from nothing else.
|
||||
test("the runner's totals are read from its summary", () => {
|
||||
const said = counted("✔ something (1ms)\nℹ tests 22\nℹ pass 22\nℹ fail 0\n");
|
||||
assert.deepEqual(said, { passed: 22, failed: 0 });
|
||||
assert.deepEqual(said, { passed: 22, failed: 0, skipped: null });
|
||||
});
|
||||
|
||||
// A test *named* like a total must not be mistaken for one. The summary is a line of its own, and
|
||||
@@ -18,7 +18,7 @@ test("a test named like a total is not a total", () => {
|
||||
// A failing run is read as a failing run.
|
||||
test("failures are read", () => {
|
||||
const said = counted("ℹ pass 21\nℹ fail 1\n");
|
||||
assert.deepEqual(said, { passed: 21, failed: 1 });
|
||||
assert.deepEqual(said, { passed: 21, failed: 1, skipped: null });
|
||||
});
|
||||
|
||||
// **No totals is not zero failures.** A run whose result could not be read is a run nobody can say
|
||||
@@ -36,7 +36,7 @@ test("output with no summary yields no totals rather than a clean bill", () => {
|
||||
// check it — 04-ISSUES/005 in miniature, inside the fix for it.
|
||||
test("a run whose result could not be read writes nothing", () => {
|
||||
let wrote = false;
|
||||
const said = reportOn({ passed: null, failed: null }, () => {
|
||||
const said = reportOn({ passed: null, failed: null }, [], () => {
|
||||
wrote = true;
|
||||
return { against: {}, ran: [] };
|
||||
});
|
||||
@@ -46,12 +46,13 @@ test("a run whose result could not be read writes nothing", () => {
|
||||
|
||||
test("a run that was read is recorded, with what it was read against", () => {
|
||||
let got: [number, number] | null = null;
|
||||
const said = reportOn({ passed: 22, failed: 0 }, (p, f) => {
|
||||
const said = reportOn({ passed: 22, failed: 0 }, [], (p, f) => {
|
||||
got = [p, f];
|
||||
return { against: { "mesh-lab": "abc1234" }, ran: ["test/integration/mesh.test.ts"] };
|
||||
});
|
||||
assert.deepEqual(got, [22, 0]);
|
||||
assert.match(said, /mesh\.test\.ts — 22 passed, 0 failed, against mesh-lab abc1234/);
|
||||
assert.match(said, /mesh\.test\.ts — 22 passed, 0 failed, 0 skipped, against mesh-lab abc1234/);
|
||||
assert.doesNotMatch(said, /NOT PASSED/);
|
||||
});
|
||||
|
||||
// The runner's real output, colours and all.
|
||||
@@ -64,5 +65,5 @@ test("the runner's totals are read from output as it actually arrives", () => {
|
||||
"\u001b[34m\u2139 pass 22\u001b[39m\n" +
|
||||
"\u001b[34m\u2139 fail 0\u001b[39m\n" +
|
||||
"\u001b[34m\u2139 duration_ms 98.9\u001b[39m\n";
|
||||
assert.deepEqual(counted(real), { passed: 22, failed: 0 });
|
||||
assert.deepEqual(counted(real), { passed: 22, failed: 0, skipped: null });
|
||||
});
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { uplinkAddress } from "../src/lifecycle/raise.ts";
|
||||
|
||||
// A workstation behind a VPN client that routes every private range leaves incus nothing to pick, so
|
||||
// the uplink's range may be named; otherwise incus picks it.
|
||||
test("the uplink's range is incus's to pick unless the host names one", () => {
|
||||
assert.equal(uplinkAddress({}), "auto");
|
||||
assert.equal(uplinkAddress({ MESH_LAB_UPLINK_V4: " 192.168.231.1/24 " }), "192.168.231.1/24");
|
||||
});
|
||||
Reference in New Issue
Block a user