package modules import ( "encoding/json" "fmt" "os" "path/filepath" "regexp" "slices" "strings" "testing" "github.com/novox/mesh-control/internal/catalogue" "github.com/novox/mesh-control/internal/overlay" ) // The examples are manifests, so the thing to check is that the catalogue accepts them. // // A manifest that only ever appears in a document is a manifest nobody has run through the parser, // and the parser refuses unknown keys — so a typo here would be discovered by whoever first tried // to use one, which is the opposite of what an example is for. func read(t *testing.T, name string) catalogue.Manifest { t.Helper() raw, err := os.ReadFile(filepath.Join(".", name)) if err != nil { t.Fatal(err) } m, err := catalogue.ParseManifest(raw) if err != nil { t.Fatalf("%s is not a manifest this mesh accepts: %v", name, err) } return m } func TestEveryExampleIsAManifestTheMeshAccepts(t *testing.T) { found, err := filepath.Glob("*.json") if err != nil { t.Fatal(err) } if len(found) == 0 { t.Fatal("no examples, so this test proves nothing") } for _, name := range found { read(t, name) } } // The serving module reads what the mesh writes, and restarts when the mesh rewrites it. // // Without the second it would serve the names it started with for ever — every machine that // joined afterwards unreachable by name, and every check passing. func TestTheResolverReadsTheMeshsNamesAndFollowsThem(t *testing.T) { m := read(t, "dnsmasq.json") var config, service map[string]any for _, r := range m.Resources { switch r["id"] { case "config": config = r case "service": service = r } } if config == nil || service == nil { t.Fatal("the module has no configuration or no service") } if !strings.Contains(config["content"].(string), overlay.ResolverPath) { t.Fatalf("it does not read what the mesh writes at %s", overlay.ResolverPath) } var follows bool for _, id := range service["restart-on"].([]any) { if id.(string) == overlay.Resolver+".nodes" { follows = true } } if !follows { t.Fatalf("it does not restart when the mesh rewrites the names: %v", service["restart-on"]) } } // It binds names the mesh chose, so it needs to know nothing about the machine it is on. // // That is the whole reason these can be static manifests: a resolver must bind somewhere and a // stub must be pointed somewhere, and neither address is knowable in advance — unless the mesh // named it. func TestTheResolverNeedsToKnowNothingAboutItsMachine(t *testing.T) { config := read(t, "dnsmasq.json").Resources[1]["content"].(string) // The directive, not the word: the comment above it names the interface too, so a plain // Contains passes whatever the module actually binds. It did. if !strings.Contains(config, "interface="+overlay.Interface+"\n") { t.Fatalf("it does not bind the private network's interface %q:\n%s", overlay.Interface, config) } // That it binds one, not which. Which address it is belongs in the manifests, where the // asking modules can be checked against it — naming it here too would be a fourth place to // keep in step, and the one nobody would think to change. if !strings.Contains(config, "\nlisten-address=127.0.0.") { t.Fatalf("it answers on no address for the machine's own use:\n%s", config) } // Not an address that belongs to something else. // // **systemd-resolved holds .53 AND .54** — the stub and the proxy stub. This module asserted // .54 was free, in a comment that read as reasoned, and a machine said otherwise: dnsmasq // could not start at all. A unit test cannot know which addresses a machine has spare, but it // can hold on to what one has already told us. for _, taken := range []string{"127.0.0.1", "127.0.0.53", "127.0.0.54"} { if strings.Contains(config, "listen-address="+taken) { t.Fatalf("it takes %s, which belongs to something else:\n%s", taken, config) } } } // Everything that points resolution at the mesh points at the same place. // // Three files name this address — one binds it and two send queries to it — and a change to one // of them alone is a resolver answering where nobody asks. func TestTheAskingModulesPointAtWhereTheResolverAnswers(t *testing.T) { serving := read(t, "dnsmasq.json").Resources[1]["content"].(string) var at string for _, line := range strings.Split(serving, "\n") { if rest, found := strings.CutPrefix(strings.TrimSpace(line), "listen-address="); found { at = rest } } if at == "" { t.Fatal("the resolver binds no address for the machine's own use") } for _, asking := range []string{"resolved-split-dns.json", "resolv-conf.json"} { m := read(t, asking) var mentions bool for _, r := range m.Resources { if content, ok := r["content"].(string); ok && strings.Contains(content, at) { mentions = true } } if !mentions { t.Fatalf("%s does not point at %s, where the resolver answers", asking, at) } } } // The two ways of deciding what a machine asks claim the same thing, so the mesh refuses the pair. func TestTwoWaysOfOwningTheResolverCannotBothBeAssigned(t *testing.T) { shelf := map[string]catalogue.Manifest{} for _, name := range []string{"resolved-split-dns.json", "resolv-conf.json", "dnsmasq.json"} { m := read(t, name) shelf[m.Module] = m } // Something has to answer `wildcard-resolution`, or they are refused for that instead and the // test would pass without ever reaching the claim. shelf["dnsmasq"] = read(t, "dnsmasq.json") _, err := catalogue.Resolve(shelf, []string{"dnsmasq", "resolved-split-dns", "resolv-conf"}, catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, catalogue.World{Unchecked: true}) if err == nil { t.Fatal("both ways of owning the resolver were assigned to one machine") } said := err.Error() if !strings.Contains(said, "the-resolver-configuration") { t.Fatalf("the refusal does not name what they both want: %v", said) } } // And the two roles are not the same claim: a machine runs one resolver AND one thing deciding // what it asks, so serving and asking must be assignable together. func TestServingAndAskingAreAssignableTogether(t *testing.T) { shelf := map[string]catalogue.Manifest{} for _, name := range []string{"dnsmasq.json", "resolved-split-dns.json"} { m := read(t, name) shelf[m.Module] = m } if _, err := catalogue.Resolve(shelf, []string{"dnsmasq", "resolved-split-dns"}, catalogue.Node{Name: "anchor", Capabilities: map[string]bool{}}, catalogue.World{Unchecked: true}); err != nil { t.Fatalf("a resolver and the thing pointing at it cannot both be assigned: %v", err) } } // The examples are JSON a person edits, so a stray comma is worth catching here rather than on a // machine. func TestTheExamplesAreWellFormed(t *testing.T) { found, _ := filepath.Glob("*.json") for _, name := range found { raw, err := os.ReadFile(name) if err != nil { t.Fatal(err) } var any map[string]any if err := json.Unmarshal(raw, &any); err != nil { t.Fatalf("%s is not JSON: %v", name, err) } } } // The resolver must not look up its own upstreams. // // Whatever points a machine at the mesh writes that address into resolv.conf, so a resolver that // read it would find itself — and every query it could not answer locally would loop until its // receive queue filled. It did: 15KB of queries backed up and every lookup on the machine hung. // // It needs no upstream because it is never asked for anything else: the asking module routes only // the mesh's suffix here and leaves the rest where the machine already sent it. func TestTheResolverDoesNotAskItselfForUpstreams(t *testing.T) { config := read(t, "dnsmasq.json").Resources[1]["content"].(string) if !strings.Contains(config, "\nno-resolv\n") { t.Fatalf("it reads resolv.conf for upstreams, which now points at itself:\n%s", config) } // And names no upstream of its own: choosing one would send every query this machine cannot // answer somewhere nobody agreed to. for _, line := range strings.Split(config, "\n") { if strings.HasPrefix(strings.TrimSpace(line), "server=") { t.Fatalf("it forwards to %q, which is not the mesh's to choose", line) } } } // The two halves of an object-store edge, as a pair. // // `minio.json` is the provider. There were two manifests describing the same object store — the // other named `object-store.json` — which is not a choice between implementations but one module // written twice: same image, same provision, same scope. Assigning both to a node would have // collided on `s3-bucket`. // // A provider and a consumer that only ever appear separately are two manifests nobody has checked // against each other: the name one provides has to be the name the other requires, and the key a // consumer contributes has to be the one the provisioner reads. Both were got wrong while writing // them, and neither would have been caught by parsing either file alone. func TestTheObjectStoreEdgeFitsTogether(t *testing.T) { provider := read(t, "minio.json") consumer := read(t, "photos.json") const provision = "s3-bucket" var provides bool for _, offer := range provider.Provides { if offer.Name == provision { provides = true } } if !provides { t.Fatalf("the provider does not offer %q", provision) } if !strings.Contains(strings.Join(consumer.Requires, ","), provision) { t.Fatalf("the consumer does not require %q", provision) } // Where each side wants to be told. A provider that receives nowhere is a provider the mesh // writes nothing for, and a provisioner with nothing to read. if provider.Receives[provision] == "" { t.Error("the provider says nowhere to write what its consumers asked for") } if provider.Grants[provision] == "" { t.Error("the provider says nowhere to write its consumers' credentials") } if consumer.Binds[provision] == "" { t.Error("the consumer says nowhere to be told where its bucket is") } if consumer.Secrets[provision] == "" { t.Error("the consumer says nowhere to be given its key") } // The key the provisioner reads out of `values`. It looks for `bucket`, so a consumer // contributing `name` — which is what the database one contributes — resolves cleanly and // then fails on the machine with "asked for a bucket and did not name it". if _, named := consumer.Contributes[provision]["bucket"]; !named { t.Errorf("the consumer contributes %v, and the provisioner reads \"bucket\"", consumer.Contributes[provision]) } } // Every hole an example leaves for a credential can be filled from what that module declared. // // **A manifest that parses is not a manifest that works.** These say `${secret:x}` in a file and // declare `x` under `own-secrets`; if the two ever disagree the mesh refuses the whole declaration // at push time, on the machine, with the module's name and nothing else to go on. Checking it here // costs nothing and moves the answer to whoever edited the file. // // This is also the shape that was missing entirely until 2026-09-01: an own secret arrives as a // file whose whole content is the password, and every one of these programs reads `KEY=value`. The // manifests said `own-secrets` pointed at a `.env` and it did not — it pointed at a password. func TestEveryCredentialHoleCanBeFilledByTheModuleThatLeftIt(t *testing.T) { found, err := filepath.Glob("*.json") if err != nil { t.Fatal(err) } var checked int for _, name := range found { m := read(t, name) has := map[string]bool{} for own := range m.OwnSecrets { has[own] = true } for required := range m.Secrets { has[required] = true } for _, r := range m.Resources { content, ok := r["content"].(string) if !ok { continue } for _, wanted := range secretsUsedForTest(content) { checked++ if !has[wanted] { t.Errorf( "%s: %v says ${secret:%s}, and %s neither owns a secret by that name "+ "nor requires anything that grants one", name, r["id"], wanted, m.Module) } } } } if checked == 0 { t.Fatal("no example puts a credential into a file, so this test proves nothing") } } // A secret file is a password and nothing else, so nothing may read one as an env file. // // The fault this catches is the one these manifests shipped with: `own-secrets` pointing at a // path called `.env`, mounted as `env-file`, holding a bare password. Docker reads that as a // malformed line and the container starts with no password at all. func TestNoContainerReadsABarePasswordAsAnEnvFile(t *testing.T) { found, _ := filepath.Glob("*.json") for _, name := range found { m := read(t, name) bare := map[string]bool{} for _, where := range m.OwnSecrets { bare[where] = true } for _, where := range m.Secrets { bare[where] = true } for _, r := range m.Resources { files, ok := r["env-file"].([]any) if !ok { continue } for _, f := range files { if bare[fmt.Sprint(f)] { t.Errorf( "%s: %v reads %s as an env file, and that path holds a bare password — "+ "declare a file whose content says ${secret:...} and read that instead", name, r["id"], f) } } } } } // The same expression the control plane and the host both match. var placeholder = regexp.MustCompile(`\$\{secret:([a-z0-9][a-z0-9-]*)\}`) func secretsUsedForTest(content string) []string { var used []string seen := map[string]bool{} for _, m := range placeholder.FindAllStringSubmatch(content, -1) { if !seen[m[1]] { seen[m[1]] = true used = append(used, m[1]) } } return used } // Every module that requires something produces configuration a program could use. // // **Parsing is not working, and this file has now learned that twice.** These modules parsed and // resolved for a day while their credentials went into files nothing could read; they would parse // and resolve just as happily with a connection string naming no user, or with a placeholder // written through as a hostname. What has to be true is that the bytes reaching the machine are // usable, so that is what this asks — of every consumer, not of the one that was being worked on. func TestEveryConsumerGetsConfigurationAProgramCouldUse(t *testing.T) { found, err := filepath.Glob("*.json") if err != nil { t.Fatal(err) } shelf := map[string]catalogue.Manifest{} for _, name := range found { m := read(t, name) shelf[m.Module] = m } var checked int for _, m := range shelf { if len(m.Requires) == 0 { continue } out := declareOnItsOwn(t, shelf, m) if out == nil { continue } checked++ for _, r := range out { content, ok := r["content"].(string) if !ok { continue } // A placeholder written through is read as a value by whatever parses the file — a // connection to a host literally called "${bound:postgres-database:at}", failing // somewhere that names neither the module nor the mesh. if strings.Contains(content, "${bound:") { t.Errorf("%s: %v reached the machine with a placeholder in it:\n%s", m.Module, r["id"], content) } // The password is the one that must survive: only the host may fill it, and only on // the machine. If it is gone, something composed it here. for _, line := range strings.Split(content, "\n") { if strings.Contains(line, "PASSWORD") || strings.Contains(line, "PASSWD") { if !strings.Contains(line, "${secret:") { t.Errorf("%s: %v carries %q, which is not a hole the host fills", m.Module, r["id"], line) } } } } } if checked == 0 { t.Fatal("no example requires anything, so this test proves nothing") } } // declareOnItsOwn resolves one consumer against a mesh that answers everything it requires, and // returns what would reach the machine. Nil when its requirements cannot be answered from the // examples, which is not this test's business to complain about. func declareOnItsOwn(t *testing.T, shelf map[string]catalogue.Manifest, m catalogue.Manifest) []map[string]any { t.Helper() // Everything it requires, answered from somewhere else in the mesh, with whatever the // providing example says it serves. offered := map[string][]catalogue.Provider{} for _, want := range m.Requires { // Built the way the control plane builds it: what a provider tells a consumer includes // the port, and the module no longer writes that into `serves` by hand — it says it once // in `listens` and the mesh puts it there (novox/hq ADR 0038). serves := map[string]any{} for _, other := range shelf { if _, said := other.Serves[want]; said { serves = catalogue.ServedOn(other, want, nil) } } offered[want] = []catalogue.Provider{ {Node: "anchor", At: "anchor.internal", Serves: serves}} } resolved, err := catalogue.Resolve(shelf, []string{m.Module}, catalogue.Node{Name: "workstation", At: "workstation.internal", Capabilities: map[string]bool{"container-runtime": true}}, catalogue.World{Offered: offered}) if err != nil { t.Logf("%s does not resolve on its own: %v", m.Module, err) return nil } for i := range resolved.Needs { resolved.Needs[i].Sealed = "sealed" } own := map[string]map[string]string{} for name := range m.OwnSecrets { if own[m.Module] == nil { own[m.Module] = map[string]string{} } own[m.Module][name] = "sealed" } out, err := resolved.Declaration(catalogue.Rendering{Needed: own}) if err != nil { t.Errorf("%s resolves and does not declare: %v", m.Module, err) return nil } return out } // Every image an example names is one this repository builds. // // A manifest naming an image nothing produces is a module that resolves, plans, pushes, and stops // on the machine at `docker pull` — the fault arriving as far from its cause as it can get. Two of // these were found by reading the manifests rather than by running them: the object store's // provisioner had a Dockerfile and no target, and Keycloak's did not exist at all. // // Only the mesh's own images are checked. `postgres`, `redis` and the rest come from a registry // and are somebody else's to build; what this bounds is the set this repository is responsible // for and might forget. func TestEveryImageTheExamplesNameIsOneThisRepositoryBuilds(t *testing.T) { makefile, err := os.ReadFile(filepath.Join("..", "..", "Makefile")) if err != nil { t.Fatal(err) } found, _ := filepath.Glob("*.json") var checked int for _, name := range found { for _, r := range read(t, name).Resources { image, ok := r["image"].(string) if !ok { continue } repository, _, _ := strings.Cut(image, "@") if !strings.HasPrefix(repository, "mesh-") { continue } checked++ if !strings.Contains(string(makefile), repository+":") { t.Errorf( "%s names the image %q and nothing in this repository builds one. A module "+ "naming an image that does not exist resolves, plans, pushes, and stops "+ "on the machine at `docker pull`", name, repository) } } } if checked == 0 { t.Fatal("no example names an image this repository builds, so this proves nothing") } } // Every host path a container mounts is a directory the module declared. // // **The mesh owns a directory or it does not** (novox/hq 04-ISSUES/026). A bind mount whose source // does not exist is created by the container runtime as root, with a mode nobody chose — so // `owner` and `mode` go unapplied on exactly the directories that hold the data. // // Worse, the rule that a directory is *kept* rather than removed when it holds something the mesh // did not put there (ADR 0030) is written in terms of declared directories. An undeclared one is // not covered by it. So the single rule guarding against data loss reached the configuration and // not the data. // // These manifests were written by carrying compose files across, and a container shape that can // express a compose file gets filled in like one. This is the check that says so. func TestEveryMountedPathIsADirectoryTheModuleDeclared(t *testing.T) { found, _ := filepath.Glob("*.json") var checked int for _, name := range found { m := read(t, name) declared := map[string]bool{} for _, r := range m.Resources { if fmt.Sprint(r["type"]) == "directory" { declared[fmt.Sprint(r["path"])] = true } } for _, r := range m.Resources { for _, v := range stringsOfTest(r["volumes"]) { host, _, _ := strings.Cut(v, ":") if !strings.HasPrefix(host, "/") { continue // a named volume, which the runtime owns and the mesh does not } checked++ // A machine facility is not the module's data, and the manifest cannot yet say // so (novox/hq 04-ISSUES/026, reopened on exactly this): the runtime's socket // exists, the machine owns it, and declaring it as the module's directory would // be a lie the host acts on. Named here one by one rather than waved through by // pattern, so each new facility is a deliberate addition beside the issue that // owns the vocabulary. if host == "/var/run/docker.sock" { continue } var covered bool for d := range declared { if host == d || strings.HasPrefix(host, strings.TrimRight(d, "/")+"/") { covered = true } } if !covered { t.Errorf( "%s: %v mounts %s and no resource declares it. The runtime will create it "+ "as root, and the rule that keeps a directory holding data does not "+ "reach a directory the mesh never declared", name, r["id"], host) } } } } if checked == 0 { t.Fatal("no example mounts a host path, so this test proves nothing") } } func stringsOfTest(v any) []string { list, ok := v.([]any) if !ok { return nil } out := make([]string, 0, len(list)) for _, item := range list { out = append(out, fmt.Sprint(item)) } return out } // A resource uses only the keys its shape has. // // **The host is the only thing that knew, and it is five steps downstream.** A container carrying // `restart-on` — which belongs to a service — composed into a declaration without complaint, was // pushed, and was refused on the machine. The host refused *the whole declaration*, correctly, // because applying the parts it understood would leave a machine that looks configured and is // not. So one misplaced key stopped a module dead, and the only place that said so was a log on a // lab machine after a seventeen-minute run. // // Nine of them had shipped across seven modules. // // The lists are written out rather than imported: the host is another repository and this is its // wire format, like the shape of a grant file. Duplicated deliberately, and checked — a contract // with two copies and no check is a contract until somebody edits one. func TestAResourceUsesOnlyTheKeysItsShapeHas(t *testing.T) { common := []string{"id", "type"} shapes := map[string][]string{ "file": {"path", "content", "bytes", "sealed", "secrets", "mode", "owner"}, "directory": {"path", "mode", "owner"}, "container": {"name", "image", "env", "env-file", "ports", "volumes", "args", "hosts", "network", "artifact"}, "service": {"unit", "state", "boot", "restart-on"}, "package": {"package", "state"}, "network": {"name"}, "archive": {"path", "artifact", "digest", "owner", "mode"}, "user": {"name", "shell", "groups", "home"}, "action": {"command", "verify", "in"}, } found, _ := filepath.Glob("*.json") var checked int for _, name := range found { for _, r := range read(t, name).Resources { kind := fmt.Sprint(r["type"]) allowed, known := shapes[kind] if !known { t.Errorf("%s: %v is a %q, which is not a shape the mesh has", name, r["id"], kind) continue } for key := range r { checked++ // `merge` and `protected` are read by the control plane and removed before a // machine sees them, so they are legal here and unknown to the host. if key == "merge" || key == "protected" { continue } if !slices.Contains(common, key) && !slices.Contains(allowed, key) { t.Errorf( "%s: %v is a %s and carries %q, which that shape does not have. It would "+ "compose cleanly and be refused on the machine — and the host refuses "+ "the whole declaration, so this stops the module entirely", name, r["id"], kind, key) } } } } if checked == 0 { t.Fatal("no example declares a resource, so this test proves nothing") } }