package asks import ( "bytes" "context" "encoding/json" "fmt" "net" "os" "os/exec" "path/filepath" "regexp" "strings" "sync" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) // machine is the lab's one machine, as the controller's fixture names it. const machine = "anchor" // credential is what a lab process connects as (the controller fixture's credentials.json, mesh-tools' // bus.Credential). type credential struct { URL string `json:"url"` Node string `json:"node,omitempty"` Module string `json:"module,omitempty"` User string `json:"user"` Password string `json:"password"` } // lab is one run of the proof: the repositories it builds from, its directory, its bus, its phone. type lab struct { t *testing.T repos clones dir string url string creds map[string]credential server *server.Server // observer is the lab's own user on the bus, outside the mesh's composition: it plays the controller's // conditions verb and reads the router's state, and is never used to show that something is allowed. observer *nats.Conn tg *fakeTelegram mu sync.Mutex conditions []map[string]any // notFree are the machines the controller's root-free verb answers as not root-free. notFree map[string]string rootAsked int desk []string // what the desk was shown procs []*exec.Cmd logs map[string]*bytes.Buffer } // clones is where each core repository the proof builds is checked out: side by side in MESH_LAB_REPOS (beside // mesh-lab by default), or one by one in MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and MESH_LAB_CATALOGUE — a branch // under review is often a worktree of its own. type clones struct{ controller, tools, catalogue string } func clonesFor(t *testing.T) clones { repos := os.Getenv("MESH_LAB_REPOS") if repos == "" { repos = filepath.Join("..", "..") } at := func(env, name string) string { dir := os.Getenv(env) if dir == "" { dir = filepath.Join(repos, name) } abs, err := filepath.Abs(dir) if err != nil { t.Fatal(err) } return abs } c := clones{controller: at("MESH_LAB_CONTROLLER", "mesh-controller"), tools: at("MESH_LAB_TOOLS", "mesh-tools"), catalogue: at("MESH_LAB_CATALOGUE", "mesh-catalog")} for _, want := range []string{filepath.Join(c.controller, "go.mod"), filepath.Join(c.tools, "node-tools", "go.mod"), filepath.Join(c.catalogue, "modules", "messenger", "go.mod"), filepath.Join(c.catalogue, "modules", "telegram", "go.mod")} { if _, err := os.Stat(want); err != nil { t.Skipf("NOT RUN: the proof builds the controller, the runtime, the router and the Telegram channel from "+ "their clones, and there is no %s (set MESH_LAB_REPOS, or MESH_LAB_CONTROLLER, MESH_LAB_TOOLS and "+ "MESH_LAB_CATALOGUE)", want) } } // The controller beside is, on the build seat, the one the mesh runs: until it carries the proof's fixture // (mesh-controller's TestTheAsksLabBus), the proof cannot compose its bus, and says so rather than fail // every pull request of the lab for a merge not yet made. fixtures, _ := filepath.Glob(filepath.Join(c.controller, "internal", "inventory", "*_test.go")) has := false for _, f := range fixtures { if body, err := os.ReadFile(f); err == nil && strings.Contains(string(body), "func TestTheAsksLabBus(") { has = true } } if !has { t.Skipf("NOT RUN: the controller at %s has no TestTheAsksLabBus, the fixture that composes the proof's bus "+ "(mesh-controller #157): the proof runs once the controller it is given carries it", c.controller) } return c } // run runs a command in a directory and fails the test with what it said. func (l *lab) run(dir string, env []string, name string, args ...string) string { l.t.Helper() cmd := exec.Command(name, args...) cmd.Dir = dir cmd.Env = append(os.Environ(), env...) out, err := cmd.CombinedOutput() if err != nil { l.t.Fatalf("%s %s in %s: %v\n%s", name, strings.Join(args, " "), dir, err, out) } return string(out) } // compose asks the controller at its commit for the bus it would compose, then (bus set) to raise it. func (l *lab) compose(bus string) { // The anchor composed as a machine the controller measured root-free: verified-sender reaches the router // only then (ADR 0259 §8). The not-free case is played live, through the root-free verb, in the proof. env := []string{"GOFLAGS=-mod=vendor", "GOPROXY=off", "MESH_LAB_ASKS_OUT=" + l.dir, "MESH_LAB_ASKS_ROOT_FREE=true", "MESH_LAB_ASKS_CATALOGUE=" + filepath.Join(l.repos.catalogue, "modules"), "MESH_LAB_ASKS_RUNTIME=" + filepath.Join(l.repos.tools, "node-tools", "module.json")} if bus != "" { env = append(env, "MESH_LAB_ASKS_BUS="+bus) } out := l.run(l.repos.controller, env, "go", "test", "-count=1", "-v", "-run", "^TestTheAsksLabBus$", "./internal/inventory/") if !strings.Contains(out, "--- PASS: TestTheAsksLabBus") { l.t.Fatalf("the controller did not compose the lab's bus (is its fixture on this commit?):\n%s", out) } } // startBus starts a server of the mesh's release with the controller's composed accounts, and the lab's // observer beside them. func (l *lab) startBus() { l.t.Helper() accounts, err := os.ReadFile(filepath.Join(l.dir, "accounts.conf")) if err != nil { l.t.Fatal(err) } const usersOpen = " users = [\n" if !bytes.Contains(accounts, []byte(usersOpen)) { l.t.Fatalf("the composed accounts block has no users list:\n%s", accounts) } observerPassword := fmt.Sprintf("observer-%d", time.Now().UnixNano()) accounts = bytes.Replace(accounts, []byte(usersOpen), []byte(usersOpen+fmt.Sprintf(" { user: \"lab-observer\", password: %q }\n", observerPassword)), 1) port := freePort(l.t) conf := fmt.Sprintf("listen: \"127.0.0.1:%d\"\njetstream {\n store_dir: %q\n}\n\n%s", port, filepath.Join(l.dir, "jetstream"), accounts) path := filepath.Join(l.dir, "bus.conf") if err := os.WriteFile(path, []byte(conf), 0o600); err != nil { l.t.Fatal(err) } opts, err := server.ProcessConfigFile(path) if err != nil { l.t.Fatalf("the composed configuration is not one the server reads: %v", err) } // The server's own log, in the lab's directory: a refusal it says is evidence, kept with the run. opts.NoSigs, opts.LogFile = true, filepath.Join(l.dir, "bus.log") opts.Trace = os.Getenv("MESH_LAB_ASKS_TRACE") != "" s, err := server.NewServer(opts) if err != nil { l.t.Fatal(err) } s.ConfigureLogger() go s.Start() if !s.ReadyForConnections(30 * time.Second) { l.t.Fatal("the lab's bus did not come up") } l.server, l.url = s, s.ClientURL() l.t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() }) raw, err := os.ReadFile(filepath.Join(l.dir, "credentials.json")) if err != nil { l.t.Fatal(err) } if err := json.Unmarshal(raw, &l.creds); err != nil { l.t.Fatal(err) } l.observer, err = nats.Connect(l.url, nats.UserInfo("lab-observer", observerPassword)) if err != nil { l.t.Fatal(err) } l.t.Cleanup(l.observer.Close) } func freePort(t *testing.T) int { ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer ln.Close() return ln.Addr().(*net.TCPAddr).Port } // as connects as one of the composed users. refused collects what the server refused it. func (l *lab) as(user string) (*nats.Conn, *refusals) { l.t.Helper() c, ok := l.creds[user] if !ok { l.t.Fatalf("the controller composed no user %s", user) } r := &refusals{} nc, err := nats.Connect(l.url, nats.UserInfo(c.User, c.Password), nats.CustomInboxPrefix("_INBOX."+c.User), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { r.add(err) })) if err != nil { l.t.Fatalf("connecting as %s: %v", user, err) } l.t.Cleanup(nc.Close) return nc, r } // refusals are the permission violations the server said to one connection. type refusals struct { mu sync.Mutex said []string } func (r *refusals) add(err error) { r.mu.Lock() defer r.mu.Unlock() r.said = append(r.said, err.Error()) } func (r *refusals) of(subject string) bool { r.mu.Lock() defer r.mu.Unlock() for _, s := range r.said { if strings.Contains(strings.ToLower(s), "permissions violation") && strings.Contains(s, subject) { return true } } return false } // refusedToPublish publishes once as a user and says whether the server refused it. func (l *lab) refusedToPublish(user, subject string, body []byte) bool { l.t.Helper() nc, r := l.as(user) if err := nc.Publish(subject, body); err != nil { return true } _ = nc.Flush() deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if r.of(subject) { return true } time.Sleep(50 * time.Millisecond) } return false } // build builds one Go program from a directory into the lab's bin. func (l *lab) build(dir, pkg, name string, ldflags string) string { out := filepath.Join(l.dir, "bin", name) args := []string{"build", "-o", out} if ldflags != "" { args = append(args, "-ldflags", ldflags) } l.run(dir, []string{"GOPRIVATE=git.novox.be", "GOFLAGS=-mod=mod", "CGO_ENABLED=0"}, "go", append(args, pkg)...) return out } // runtime starts the mesh's runtime on a module's own credential, serving that module's bundle alone: how // the mesh runs a module that `runs-as` an account of its own. func (l *lab) runtime(module, runtimeBin, bundle string, env map[string]string) { l.t.Helper() user := machine + "." + module c := l.creds[user] c.URL, c.Node, c.Module = l.url, machine, module credPath := filepath.Join(l.dir, module, "broker") if err := os.MkdirAll(filepath.Dir(credPath), 0o700); err != nil { l.t.Fatal(err) } raw, _ := json.Marshal(c) if err := os.WriteFile(credPath, raw, 0o600); err != nil { l.t.Fatal(err) } toolEnv, _ := json.Marshal(map[string]map[string]string{module: env}) cmd := exec.Command(runtimeBin, "serve") cmd.Env = append(os.Environ(), "MESH_BROKER_FILE="+credPath, "MESH_TOOL_MODULES="+module+"="+bundle, "MESH_TOOL_ENV="+string(toolEnv)) logs := &bytes.Buffer{} cmd.Stdout, cmd.Stderr = &lockedWriter{w: logs, mu: &l.mu}, &lockedWriter{w: logs, mu: &l.mu} if err := cmd.Start(); err != nil { l.t.Fatal(err) } l.mu.Lock() l.procs = append(l.procs, cmd) if l.logs == nil { l.logs = map[string]*bytes.Buffer{} } l.logs[module] = logs l.mu.Unlock() l.t.Cleanup(func() { _ = cmd.Process.Signal(os.Interrupt) done := make(chan struct{}) go func() { _ = cmd.Wait(); close(done) }() select { case <-done: case <-time.After(5 * time.Second): _ = cmd.Process.Kill() } if l.t.Failed() || os.Getenv("MESH_LAB_ASKS_SAY") != "" { l.mu.Lock() l.t.Logf("--- what the %s runtime said:\n%s", module, logs.String()) l.mu.Unlock() } }) } // said is what a runtime said so far. func (l *lab) said(module string) string { l.mu.Lock() defer l.mu.Unlock() if b := l.logs[module]; b != nil { return b.String() } return "" } // waitSaid waits for a runtime to say something. func (l *lab) waitSaid(module, what string, within time.Duration) { l.t.Helper() deadline := time.Now().Add(within) for time.Now().Before(deadline) { if strings.Contains(l.said(module), what) { return } time.Sleep(200 * time.Millisecond) } l.t.Fatalf("the %s runtime never said %q within %s:\n%s", module, what, within, l.said(module)) } type lockedWriter struct { w *bytes.Buffer mu *sync.Mutex } func (w *lockedWriter) Write(p []byte) (int, error) { w.mu.Lock() defer w.mu.Unlock() return w.w.Write(p) } // playController answers the controller's `conditions` verb from the lab's list, and its `root-free` verb from // the lab's word on each machine (free unless notFree names it), as the router asks them. That a machine is // root-free is the controller's own judgement, proven in its repository (probe_agent_root_test.go); here the // lab says it, so the proof can show what the router does with each answer. func (l *lab) playController() { l.t.Helper() free, err := l.observer.Subscribe("mesh.seat.mesh-controller.tool.root-free", func(m *nats.Msg) { var args struct { Machines []string `json:"machines"` } _ = json.Unmarshal(m.Data, &args) var out []map[string]any l.mu.Lock() for _, name := range args.Machines { why, not := l.notFree[name] if !not { why = "the lab says no agent can become root here" } out = append(out, map[string]any{"machine": name, "free": !not, "why": why, "judged": time.Now().UTC().Format(time.RFC3339Nano)}) } l.rootAsked++ l.mu.Unlock() raw, _ := json.Marshal(map[string]any{"result": map[string]any{"machines": out}, "node": machine}) _ = m.Respond(raw) }) if err != nil { l.t.Fatal(err) } l.t.Cleanup(func() { _ = free.Unsubscribe() }) sub, err := l.observer.Subscribe("mesh.seat.mesh-controller.tool.conditions", func(m *nats.Msg) { l.mu.Lock() list := append([]map[string]any{}, l.conditions...) l.mu.Unlock() raw, _ := json.Marshal(map[string]any{"result": map[string]any{"conditions": list}, "node": machine}) _ = m.Respond(raw) }) if err != nil { l.t.Fatal(err) } l.t.Cleanup(func() { _ = sub.Unsubscribe() }) } // raise says a condition as the controller does: on its event, and in its conditions verb from then on. func (l *lab) raise(c map[string]any) { l.t.Helper() l.mu.Lock() l.conditions = append(l.conditions, c) l.mu.Unlock() l.sayCondition("condition-raised", c) } // clear ends a condition as the controller does. func (l *lab) clear(key string) { l.t.Helper() l.mu.Lock() var kept []map[string]any var gone map[string]any for _, c := range l.conditions { if c["key"] == key { gone = c continue } kept = append(kept, c) } l.conditions = kept l.mu.Unlock() if gone != nil { gone["cleared"] = time.Now().UTC().Format(time.RFC3339Nano) l.sayCondition("condition-cleared", gone) } } func (l *lab) sayCondition(event string, c map[string]any) { l.t.Helper() nc, _ := l.as("controller") js, err := nc.JetStream() if err != nil { l.t.Fatal(err) } raw, _ := json.Marshal(c) msg := nats.NewMsg("mesh.seat.mesh-controller.event." + event) msg.Data = raw msg.Header.Set("x-event-id", fmt.Sprintf("%s.%v.%d", event, c["key"], time.Now().UnixNano())) if _, err := js.PublishMsg(msg); err != nil { l.t.Fatalf("saying %s as the controller: %v", event, err) } } // playDesk is the desk channel (kind `desktop`) at the bus, on the machine's runtime's credential, which // carries the desk channel: it says it is ready, and keeps what the router shows it — the link's code. func (l *lab) playDesk(ctx context.Context) { l.t.Helper() // The desk channel runs as an account of its own (it carries a link's code), on its own credential. nc, _ := l.as(machine + ".desk-channel") stand := func() { raw, _ := json.Marshal(map[string]any{"ready": true, "at": time.Now().UTC()}) js, _ := nc.JetStream() msg := nats.NewMsg("mesh.seat.intake.event.standing.desktop") msg.Data = raw msg.Header.Set("x-event-id", fmt.Sprintf("standing.desktop.%d", time.Now().UnixNano())) if _, err := js.PublishMsg(msg); err != nil { l.t.Logf("the desk could not say it is ready: %v", err) } } stand() go func() { tick := time.NewTicker(20 * time.Second) defer tick.Stop() for { select { case <-ctx.Done(): return case <-tick.C: stand() } } }() js, err := jetstream.New(nc) if err != nil { l.t.Fatal(err) } bind, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() worker, err := js.Consumer(bind, "SEAT_CHANNEL", "SEAT_CHANNEL_DESKTOP_worker") if err != nil { l.t.Fatalf("the desk's worker: %v", err) } consuming, err := worker.Consume(func(m jetstream.Msg) { var shown struct { Title string `json:"title"` Body string `json:"body"` } _ = json.Unmarshal(m.Data(), &shown) l.mu.Lock() l.desk = append(l.desk, shown.Title+": "+shown.Body) l.mu.Unlock() _ = m.Ack() }) if err != nil { l.t.Fatal(err) } l.t.Cleanup(consuming.Stop) } var linkCode = regexp.MustCompile(`(\d{3}) (\d{3})\.`) // codeOnTheDesk waits for the link's code the router shows on the desk. func (l *lab) codeOnTheDesk(within time.Duration) string { l.t.Helper() deadline := time.Now().Add(within) for time.Now().Before(deadline) { l.mu.Lock() for _, s := range l.desk { if m := linkCode.FindStringSubmatch(s); m != nil { l.mu.Unlock() return m[1] + m[2] } } l.mu.Unlock() time.Sleep(100 * time.Millisecond) } l.mu.Lock() defer l.mu.Unlock() l.t.Fatalf("the desk was never shown a code; it was shown %q\nthe router said:\n%s", l.desk, l.logs["messenger"]) return "" } // routerRecord reads one key of the router's state, as the lab's observer. func (l *lab) routerRecord(bucket, key string) (jetstream.KeyValueEntry, jetstream.KeyValue) { l.t.Helper() js, err := jetstream.New(l.observer) if err != nil { l.t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() kv, err := js.KeyValue(ctx, bucket) if err != nil { l.t.Fatalf("the router's bucket %s: %v", bucket, err) } e, err := kv.Get(ctx, key) if err != nil { l.t.Fatalf("the router's %s %s: %v", bucket, key, err) } return e, kv }