diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 1a86332..c49ceca 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -171,10 +171,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri // after a clone that then fails at npm ci. built, err = builder.Build(ctx, builder.Command, publisher, request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, - forgeFrom(), - func(step, message string) { + forgeFrom(), func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) - }) + }, request.Seats) } if err != nil { // A failure is a result. A build that fails and says nothing is indistinguishable from a diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index bf2d566..7b4be87 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -408,6 +408,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti Path: path, Ref: ref, Held: heldBy(ctx), + Seats: seatBases(ctx), } fmt.Printf("asked for %s", source) if source.Seat != "" { @@ -513,7 +514,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai result, err := ask.Submit(ctx, link.BuildRequest{ ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), Repository: repository, Path: path, Ref: ref, - Held: heldBy(ctx), + Held: heldBy(ctx), Seats: seatBases(ctx), }, wait) if err != nil { return err diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index ab75fab..c0408ca 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -30,6 +30,7 @@ func moduleCheck(paths []string, out io.Writer) error { "manifest of a repository together so the rules between them are checked too") } shelf := catalogue.Shelf{} + faulted := map[string]bool{} failed := 0 for _, path := range paths { raw, err := os.ReadFile(path) @@ -50,6 +51,15 @@ func moduleCheck(paths []string, out io.Writer) error { failed++ continue } + // A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here and in the + // catalogue-wide test, not yet at registration, while the declared exceptions shrink. + if named := catalogue.InstallationProblems(m); len(named) > 0 { + for _, p := range named { + fmt.Fprintf(out, "%s: %s\n", path, p) + } + failed += len(named) + faulted[m.Module] = true + } shelf[m.Module] = m } @@ -70,6 +80,9 @@ func moduleCheck(paths []string, out io.Writer) error { sort.Strings(names) for _, name := range names { m := shelf[name] + if faulted[name] { + continue + } fmt.Fprintf(out, "%s: ok", name) if n := len(m.Tools); n > 0 { fmt.Fprintf(out, ", %d tool(s)", n) diff --git a/cmd/mesh-controller/source.go b/cmd/mesh-controller/source.go index 760dda7..c5b7be9 100644 --- a/cmd/mesh-controller/source.go +++ b/cmd/mesh-controller/source.go @@ -82,6 +82,16 @@ func cloneFrom(ctx context.Context, source buildSource) (string, error) { // serves no scheme or port has nothing to compose from — a default port here would be the forge's // address guessed, which is the thing this exists to stop. func clonedFromSeat(world catalogue.World, seatName, repository string) (string, error) { + base, err := seatBase(world, seatName) + if err != nil { + return "", err + } + path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git") + return fmt.Sprintf("%s/%s.git", base, path), nil +} + +// seatBase is `scheme://host:port` of a seat's holder as the mesh reaches it, for cloning. +func seatBase(world catalogue.World, seatName string) (string, error) { seat, known := catalogue.SeatNamed(seatName) if !known || seat.Delivers == "" { return "", fmt.Errorf("%q is not a seat a repository can live on", seatName) @@ -94,9 +104,9 @@ func clonedFromSeat(world catalogue.World, seatName, repository string) (string, } } if holder == nil { - return "", fmt.Errorf("nobody holds the %s seat, so %s cannot be cloned from this mesh's "+ + return "", fmt.Errorf("nobody holds the %s seat, so nothing can be cloned from this mesh's "+ "forge — assign a module that claims it, or build from the repository's URL without --self", - seat.Name, repository) + seat.Name) } var provider *catalogue.Provider for i, p := range world.Offered[seat.Delivers] { @@ -118,8 +128,33 @@ func clonedFromSeat(world catalogue.World, seatName, repository string) (string, return "", fmt.Errorf("%s on %s holds the %s seat and does not serve a scheme and a port for %q", holder.Module, holder.Node, seat.Name, seat.Delivers) } - path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git") - return fmt.Sprintf("%s://%s:%s/%s.git", scheme, provider.At, port, path), nil + return fmt.Sprintf("%s://%s:%s", scheme, provider.At, port), nil +} + +// seatBases is the clone base of every seat a recipe's context may name, for a build request +// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not +// name it at all, and if it does the builder refuses with the seat's name. +func seatBases(ctx context.Context) map[string]string { + open, err := openStores(ctx) + if err != nil { + return nil + } + defer open.Close() + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return nil + } + world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "") + if err != nil { + return nil + } + bases := map[string]string{} + for _, seatName := range []string{gitSeat} { + if base, err := seatBase(world, seatName); err == nil { + bases[seatName] = base + } + } + return bases } // servedPort is a served port as text, however the manifest and the node's settings carried it. diff --git a/internal/builder/builder.go b/internal/builder/builder.go index bc94d75..f247890 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -90,7 +90,13 @@ type GitCredential struct { // records — reachable, unreferenced, and indistinguishable from something in use. func Build(ctx context.Context, run Runner, publish Publisher, repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, - forge GitCredential, log Log) (Result, error) { + forge GitCredential, log Log, seats ...map[string]string) (Result, error) { + // The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers + // that hand none — tests of everything but contexts — read as they did. + var seatBases map[string]string + if len(seats) > 0 { + seatBases = seats[0] + } say := logging(log) say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref)) @@ -209,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) - made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, say) + made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say) if err != nil { say("artifact", "%s FAILED: %v", a.Name, err) return Result{}, err @@ -246,14 +252,18 @@ func logging(log Log) func(step, format string, args ...any) { // module's own repository — a fresh tree, the same way the module's own is, keyed by artifact // name so two artifacts of one module naming different contexts do not collide. func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string, - from catalogue.ArtifactContext, say func(step, format string, args ...any)) (string, error) { - say("context", "cloning %s at %s for %s", from.Repository, refOrHead(from.Ref), artifact) + from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (string, error) { + url, err := contextURL(from, seats) + if err != nil { + return "", err + } + say("context", "cloning %s at %s for %s", url, refOrHead(from.Ref), artifact) dir := filepath.Join(workspace, "context-"+artifact) if err := os.RemoveAll(dir); err != nil { return "", err } - if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", from.Repository, dir)...); err != nil { - return "", fmt.Errorf("cannot clone %s: %w", from.Repository, err) + if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil { + return "", fmt.Errorf("cannot clone %s: %w", url, err) } if from.Ref != "" { if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil { @@ -264,6 +274,23 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia return dir, nil } +// contextURL is what a context is cloned from: its URL, or — for a context on a seat — the seat's +// clone base the mesh sent with the request joined to the repository's path (novox/hq ADR 0155). +// Refused, never guessed, when the mesh sent no base for that seat: a builder that guessed a forge +// would be the literal this removes, one layer down. +func contextURL(from catalogue.ArtifactContext, seats map[string]string) (string, error) { + if from.Seat == "" { + return from.Repository, nil + } + base, told := seats[from.Seat] + if !told || base == "" { + return "", fmt.Errorf("the context is %s on the %s seat, and this build was told no clone "+ + "base for that seat — nothing holds it in this mesh, or the control plane predates the word", + from.Repository, from.Seat) + } + return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil +} + // cloneWith is a git invocation that may offer a stored credential. // // The first `-c credential.helper=` clears every helper the environment might carry, so exactly @@ -416,7 +443,8 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool { func one(ctx context.Context, run Runner, publish Publisher, module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string, - held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) { + held map[string]string, npmrc string, seats map[string]string, + say func(step, format string, args ...any)) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: @@ -493,7 +521,7 @@ func one(ctx context.Context, run Runner, publish Publisher, recipePath := a.From buildDir := tree if a.Context != nil { - cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, say) + cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, seats, say) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err) } diff --git a/internal/builder/context_test.go b/internal/builder/context_test.go new file mode 100644 index 0000000..cf7e4e9 --- /dev/null +++ b/internal/builder/context_test.go @@ -0,0 +1,26 @@ +package builder + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A context on a seat is cloned from the base the mesh sent, joined to the repository's path; a +// context by URL is itself; a seat the mesh sent no base for is refused by name (novox/hq ADR 0155). +func TestAContextOnASeatIsClonedFromTheBaseTheMeshSent(t *testing.T) { + seats := map[string]string{"git": "http://forge.example.tld:3000"} + got, err := contextURL(catalogue.ArtifactContext{Seat: "git", Repository: "org/controller"}, seats) + if err != nil || got != "http://forge.example.tld:3000/org/controller.git" { + t.Fatalf("got %q, %v", got, err) + } + got, err = contextURL(catalogue.ArtifactContext{Repository: "https://elsewhere.example/x.git"}, seats) + if err != nil || got != "https://elsewhere.example/x.git" { + t.Fatalf("a URL context was changed: %q, %v", got, err) + } + _, err = contextURL(catalogue.ArtifactContext{Seat: "git", Repository: "org/controller"}, nil) + if err == nil || !strings.Contains(err.Error(), "git seat") { + t.Fatalf("a seat with no base was not refused by name: %v", err) + } +} diff --git a/internal/catalogue/catalogue_check_test.go b/internal/catalogue/catalogue_check_test.go index 03b55db..b5ecddf 100644 --- a/internal/catalogue/catalogue_check_test.go +++ b/internal/catalogue/catalogue_check_test.go @@ -3,6 +3,7 @@ package catalogue import ( "os" "path/filepath" + "strings" "testing" ) @@ -45,3 +46,33 @@ func TestEveryCatalogueManifestParses(t *testing.T) { t.Fatal("no endpoint in the catalogue is named, so this proved nothing") } } + +// TestNoCatalogueManifestNamesAnInstallation is ADR 0112's check, run over the real catalogue: no +// definition names a domain or a public address the mesh acts on, and every value that must for now +// carries its reason (novox/hq ADR 0155, issue 134). The list it prints is the one that shrinks. +func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) { + root := os.Getenv("MESH_CATALOGUE") + if root == "" { + t.Skip("set MESH_CATALOGUE to a catalogue checkout to run this") + } + found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil || len(found) == 0 { + t.Fatalf("no manifests under %s: %v", root, err) + } + var named []string + for _, p := range found { + raw, err := os.ReadFile(p) + if err != nil { + t.Fatalf("%s: %v", p, err) + } + m, err := ParseManifest(raw) + if err != nil { + t.Errorf("%s: %v", p, err) + continue + } + named = append(named, InstallationProblems(m)...) + } + if len(named) > 0 { + t.Fatalf("%d value(s) name an installation:\n %s", len(named), strings.Join(named, "\n ")) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 5b46685..8145f1d 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -696,6 +696,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri // Said in the catalogue, not on the machine: the host parses strictly and knows no // such field, and the reason is for a reader of the manifest. delete(copied, SecretsInEnvironment) + delete(copied, NamesOnPurpose) + // **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a + // definition may not carry because it is true of one installation only. Filled from + // the same layers a mergeable file takes, and refused when no layer set it. + if err := settingInto(copied, with.Settings[m.Module], m.Module); err != nil { + return nil, err + } // **Placed before anything reads a path.** A pathless directory receives the path // this node resolves for it, and every ${dir:…} — in paths, mounts, content and // environment — becomes that path, so what follows sees only concrete places diff --git a/internal/catalogue/installation.go b/internal/catalogue/installation.go new file mode 100644 index 0000000..3068b9f --- /dev/null +++ b/internal/catalogue/installation.go @@ -0,0 +1,227 @@ +package catalogue + +import ( + "encoding/json" + "fmt" + "net" + "regexp" + "sort" + "strings" +) + +// A definition names no installation (novox/hq ADR 0112, ADR 0155, issues 122 and 134). +// +// A module definition holds what is true of the module everywhere; what is particular to one mesh — +// a public name, a forge's address, a node's public address — is resolved at assignment. The rule +// stood for a month with nothing checking it, and a sweep found thirty of seventy-one definitions +// naming the installation they were written in. This is the check. +// +// **What is judged is what the mesh acts on, not what a person reads.** A domain in a `why` or a +// `description` is documentation the mesh never reads; reporting it beside `KC_HOSTNAME` would teach +// people to ignore the report. What is judged is every other string value: a name under a public +// top-level domain, or a public address. Two families of name are the world's and not this mesh's, +// and are allowed where they can only mean the world: the public registries an `image` may be pulled +// from, and the public resolvers a machine may forward to. The container runtime's own alias for +// its host is the runtime's, true on every machine that runs it. +// +// **A name that is right where it stands is declared, one by one, with its reason.** A federated +// server's config names the federation's public directory; an application built outside the mesh +// is pulled from the registry that built it, until the mesh builds it. The resource carries +// `names-on-purpose`, a map from each such name to why — the shape `secrets-in-environment` has, +// per name — so a reader sees which names a definition means to carry and why, a name the map does +// not cover is still reported, and the catalogue-wide test is the list that shrinks as names move. + +// NamesOnPurpose is the catalogue-level word a resource carries for the names it means to name: +// each name mapped to its reason. The host never sees it. +const NamesOnPurpose = "names-on-purpose" + +// prose is every key whose value the mesh never reads. +var prose = map[string]bool{"why": true, "description": true} + +// Registries the world runs, which an image may name because an image reference must say where it +// is pulled from. Anything else in an image reference is a registry of some installation. +var worldsRegistries = map[string]bool{ + "docker.io": true, "registry-1.docker.io": true, "index.docker.io": true, "ghcr.io": true, + "quay.io": true, "gcr.io": true, "registry.k8s.io": true, "k8s.gcr.io": true, + "mcr.microsoft.com": true, "lscr.io": true, "public.ecr.aws": true, "registry.gitlab.com": true, + "codeberg.org": true, "cgr.dev": true, +} + +// Services the world runs that a definition may name as a policy default, the way it may name a +// public resolver: the public certificate authorities' ACME directories. Anything else a served +// fact or a file names is somebody's installation. +var worldsServices = map[string]bool{ + "acme-v02.api.letsencrypt.org": true, "acme-staging-v02.api.letsencrypt.org": true, + "api.buypass.com": true, "api.test4.buypass.no": true, "dv.acme-v02.api.pki.goog": true, + "acme.zerossl.com": true, +} + +// Resolvers the world runs, which a machine's resolver may forward to as a policy default. +var worldsResolvers = map[string]bool{ + "1.1.1.1": true, "1.0.0.1": true, "8.8.8.8": true, "8.8.4.4": true, "9.9.9.9": true, + "149.112.112.112": true, "208.67.222.222": true, "208.67.220.220": true, +} + +// hostname is a dotted name whose last label is a top-level domain a real installation would have. +// Not every dotted token: `module.json`, `index.html` and `docker.sock` are dotted and name nothing. +// Boundaries are checked by hand rather than in the pattern, because two names one character apart +// — `a.example.tld,b.example.tld` — would otherwise share the delimiter and the second would be lost. +var hostname = regexp.MustCompile( + `(?i)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+` + + `(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` + + `internal|example|tld|test|invalid)`) + +// address is a dotted quad. +var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`) + +// isName is whether a byte may be part of a name; a match bordered by one is a longer token. +func isName(b byte) bool { + return b == '.' || b == '-' || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') +} + +// standalone are the matches of re in value that are whole tokens, not parts of a longer one. +func standalone(re *regexp.Regexp, value string) []string { + var out []string + for _, span := range re.FindAllStringIndex(value, -1) { + if span[0] > 0 && isName(value[span[0]-1]) { + continue + } + if span[1] < len(value) && isName(value[span[1]]) { + continue + } + out = append(out, value[span[0]:span[1]]) + } + return out +} + +// InstallationProblems is every value of a definition that names an installation, in the +// definition's own words: where it is, and what it names. +func InstallationProblems(m Manifest) []string { + raw, err := json.Marshal(m) + if err != nil { + return []string{fmt.Sprintf("%s could not be read back: %v", m.Module, err)} + } + var tree any + if err := json.Unmarshal(raw, &tree); err != nil { + return []string{fmt.Sprintf("%s could not be read back: %v", m.Module, err)} + } + var problems []string + // The module's own name is a value too: a module named after the domain it serves is a + // definition that can only be installed there (issue 134). + for _, name := range namesIn(m.Module) { + problems = append(problems, fmt.Sprintf( + "%s is named after %s, and a module is named for what it is, not for where it runs", m.Module, name)) + } + walk(tree, "", nil, func(at string, value string, meant map[string]bool, isImage bool) { + for _, name := range namesIn(value) { + if (isImage && worldsRegistries[strings.ToLower(name)]) || meant[name] { + continue + } + problems = append(problems, fmt.Sprintf("%s names %s at %s", m.Module, name, at)) + } + for _, ip := range addressesIn(value) { + if meant[ip] { + continue + } + problems = append(problems, fmt.Sprintf("%s names the public address %s at %s", m.Module, ip, at)) + } + }) + sort.Strings(problems) + return problems +} + +// walk visits every string in the tree with its path, the names the enclosing resource means to +// name (with a reason), and whether it is an image reference. +func walk(node any, at string, meant map[string]bool, visit func(at, value string, meant map[string]bool, isImage bool)) { + switch v := node.(type) { + case map[string]any: + if declared, has := v[NamesOnPurpose].(map[string]any); has { + widened := map[string]bool{} + for name := range meant { + widened[name] = true + } + for name, reason := range declared { + if r, ok := reason.(string); ok && strings.TrimSpace(r) != "" { + widened[strings.ToLower(name)] = true + } + } + meant = widened + } + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if prose[k] || k == NamesOnPurpose || (at == "" && k == "module") { + continue + } + child := at + "." + k + if at == "" { + child = k + } + if s, isString := v[k].(string); isString { + visit(child, s, meant, k == "image") + continue + } + walk(v[k], child, meant, visit) + } + case []any: + for i, item := range v { + child := fmt.Sprintf("%s[%d]", at, i) + if s, isString := item.(string); isString { + visit(child, s, meant, false) + continue + } + walk(item, child, meant, visit) + } + } +} + +// namesIn is every hostname in a value that could belong to an installation. +func namesIn(value string) []string { + var out []string + for _, found := range standalone(hostname, value) { + name := strings.ToLower(found) + switch { + case strings.HasSuffix(name, ".docker.internal"): + // The container runtime's alias for its own host: every machine running it has one. + case worldsServices[name]: + // A public authority named as a policy default, true of any mesh that wants it. + case name == "example.tld", strings.HasSuffix(name, ".example.tld"), + name == "example.com", name == "example.net", name == "example.org", + strings.HasSuffix(name, ".example.com"), strings.HasSuffix(name, ".example.net"), + strings.HasSuffix(name, ".example.org"), strings.HasSuffix(name, ".example"), + strings.HasSuffix(name, ".test"), strings.HasSuffix(name, ".invalid"): + // Documentation names, which is what a definition's own example should use. + default: + out = append(out, name) + } + } + return out +} + +// addressesIn is every public address in a value: not a private range, loopback, link-local, the +// unspecified address, a documentation range, or a resolver the world runs. +func addressesIn(value string) []string { + var out []string + for _, found := range standalone(address, value) { + ip := net.ParseIP(found) + if ip == nil || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || + ip.IsUnspecified() || ip.IsMulticast() || worldsResolvers[found] || documentation(ip) { + continue + } + out = append(out, found) + } + return out +} + +func documentation(ip net.IP) bool { + for _, cidr := range []string{"192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24", "100.64.0.0/10"} { + _, block, _ := net.ParseCIDR(cidr) + if block.Contains(ip) { + return true + } + } + return false +} diff --git a/internal/catalogue/installation_test.go b/internal/catalogue/installation_test.go new file mode 100644 index 0000000..563ed72 --- /dev/null +++ b/internal/catalogue/installation_test.go @@ -0,0 +1,93 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A definition names no installation (novox/hq ADR 0112, ADR 0155). What the mesh acts on is judged; +// prose is not; the world's registries and resolvers are the world's; a declared exception is a +// reason a reader sees. +func TestADefinitionNamingAnInstallationIsNamedBack(t *testing.T) { + m := Manifest{Module: "idp", Resources: []map[string]any{ + {"id": "server", "type": "container", "image": "quay.io/keycloak/keycloak@sha256:aa", + "env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}, + {"id": "env", "type": "file", "content": "REAL_IP_FROM=192.168.1.0/24,127.0.0.0/8,203.0.113.7,51.15.22.9\n"}, + }, Listens: []Listening{{Port: 8080, From: FromMesh, Why: "the login page; login.mesh-one.be is a route grant"}}} + got := strings.Join(InstallationProblems(m), "\n") + for _, want := range []string{ + "idp names login.mesh-one.be at resources[0].env.KC_HOSTNAME", + "idp names the public address 51.15.22.9 at resources[1].content", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + for _, mustNot := range []string{"quay.io", "why", "203.0.113.7", "192.168.1.0", "127.0.0.0"} { + if strings.Contains(got, mustNot) { + t.Errorf("%q was reported and should not be:\n%s", mustNot, got) + } + } +} + +func TestTheWorldsNamesAreNotAnInstallations(t *testing.T) { + m := Manifest{Module: "resolver", Resources: []map[string]any{ + {"id": "conf", "type": "file", "content": "server=1.1.1.1\nserver=8.8.8.8\nlisten=127.0.0.55\n"}, + {"id": "proxy", "type": "container", "image": "docker.io/library/traefik@sha256:bb"}, + {"id": "adapter", "type": "file", "content": "{\"machine\": \"host.docker.internal\"}\n"}, + {"id": "doc", "type": "file", "content": "root = https://git.example.tld/\n"}, + }} + if got := InstallationProblems(m); len(got) != 0 { + t.Fatalf("the world's names were reported: %v", got) + } +} + +func TestANameMeantOnPurposeIsDeclaredWithItsReason(t *testing.T) { + // The federation's public directory in a homeserver's config: the world's, said so, and a name + // the map does not cover is still reported. + m := Manifest{Module: "homeserver", Resources: []map[string]any{ + {"id": "conf", "type": "file", "content": "trusted_key_servers: matrix.org\nwell_known: https://mesh-one.be\n", + NamesOnPurpose: map[string]any{"matrix.org": "the federation's public key server, the world's"}}, + }} + got := InstallationProblems(m) + if len(got) != 1 || !strings.Contains(got[0], "mesh-one.be") { + t.Fatalf("got %v", got) + } +} + +func TestAnImageFromAnInstallationsRegistryNeedsAReason(t *testing.T) { + bare := Manifest{Module: "site", Resources: []map[string]any{ + {"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc"}, + }} + if got := InstallationProblems(bare); len(got) != 1 || !strings.Contains(got[0], "registry.mesh-one.be") { + t.Fatalf("an image on an installation's registry was not named: %v", got) + } + excepted := Manifest{Module: "site", Resources: []map[string]any{ + {"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc", + NamesOnPurpose: map[string]any{"registry.mesh-one.be": "built outside the mesh until the site's repository is a build source here"}}, + }} + if got := InstallationProblems(excepted); len(got) != 0 { + t.Fatalf("a declared exception was still reported: %v", got) + } +} + +func TestAModuleNamedAfterADomainIsNamedBack(t *testing.T) { + got := InstallationProblems(Manifest{Module: "mesh-one.be"}) + if len(got) != 1 || !strings.Contains(got[0], "named after mesh-one.be") { + t.Fatalf("got %v", got) + } +} + +func TestABuildContextOnASeatNamesNoForge(t *testing.T) { + m := Manifest{Module: "packager", Build: &Build{Artifacts: []Artifact{ + {Name: "server", Kind: "image", From: "Dockerfile", + Context: &ArtifactContext{Seat: "git", Repository: "org/controller", Ref: "main"}}, + }}} + if got := InstallationProblems(m); len(got) != 0 { + t.Fatalf("a context on a seat was reported: %v", got) + } + m.Build.Artifacts[0].Context = &ArtifactContext{Repository: "https://git.mesh-one.be/org/controller.git"} + if got := InstallationProblems(m); len(got) != 1 { + t.Fatalf("a context by URL was not reported: %v", got) + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index db46878..044620f 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -549,8 +549,14 @@ type BuildsOn struct { type ArtifactContext struct { // Repository is cloned fresh, the same way the module's own repository is — a working tree // nothing has touched, so what was built is reproducible from the two commits named rather - // than from whatever a previous build happened to leave behind. + // than from whatever a previous build happened to leave behind. A URL, or — with Seat — a + // path on that seat's holder, `/`. Repository string `json:"repository"` + // Seat is the seat the repository lives on: `git` for this mesh's own forge (novox/hq ADR 0111, + // ADR 0155). A context written as a URL names one installation's forge and can be built + // nowhere else; a path on the seat is composed by the mesh that builds it, whichever forge + // holds the seat there. + Seat string `json:"seat,omitempty"` // Ref is the branch, tag or commit of that repository to build. Empty means its own default // branch — the same meaning an empty module ref already has. Ref string `json:"ref,omitempty"` diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go new file mode 100644 index 0000000..f63199f --- /dev/null +++ b/internal/catalogue/setting_into.go @@ -0,0 +1,109 @@ +package catalogue + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// An operator's value, where a definition needs one (novox/hq ADR 0112, ADR 0155, design 27). +// +// A mail server's domain, a site's name, the address a proxy forwards from: values that are true of +// one installation and of no other, and that a module's software must be told. They had nowhere to +// live but the definition, which is how a catalogue meant for any mesh came to name this one +// (novox/hq issues 122, 134). ADR 0112 names the operator as one of the four providers; this is the +// operator answering. +// +// `${setting:}` in a file's content is filled from the module's settings layers — the mesh's, +// then this node's — the same layers a mergeable JSON file and a contribution already take, so +// `settings set ` is the one place a person's values go. **Refused when no layer sets it**, +// naming the key and the remedy: a definition that carried a default for a mail domain would be +// carrying the very literal this removes, and a blank written silently would be a service that +// comes up wrong somewhere that names neither the module nor the key. + +// settingRef is how a definition asks for an operator's value: ${setting:}. +var settingRef = regexp.MustCompile(`\$\{setting:([a-z0-9][a-z0-9_.-]*)\}`) + +// settingsUsed is every key a file's content asks for, once each, in order of first use. +func settingsUsed(content string) []string { + var keys []string + seen := map[string]bool{} + for _, m := range settingRef.FindAllStringSubmatch(content, -1) { + if !seen[m[1]] { + seen[m[1]] = true + keys = append(keys, m[1]) + } + } + return keys +} + +// settingInto fills a file's ${setting:…} placeholders from the layers over a module. +// +// The last layer setting a key wins, which is the node's over the mesh's — the same order settle +// applies to a mergeable file. A value that is not a string is written the way a program would read +// it (a number without a trailing .000000, a boolean as true/false). +func settingInto(resource map[string]any, layers []Layer, module string) error { + if fmt.Sprint(resource["type"]) != "file" { + return nil + } + content, ok := resource["content"].(string) + if !ok { + return nil + } + for _, key := range settingsUsed(content) { + value, set := settingValue(layers, key) + if !set { + return fmt.Errorf( + "%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ + "`settings set %s ` with {%q: …}%s", + module, key, key, module, key, orNoSettings(layers)) + } + content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value)) + } + resource["content"] = content + return nil +} + +func settingValue(layers []Layer, key string) (any, bool) { + var value any + set := false + for _, layer := range layers { + if v, has := layer.Values[key]; has { + value, set = v, true + } + } + return value, set +} + +func orNoSettings(layers []Layer) string { + var keys []string + for _, l := range layers { + for k := range l.Values { + keys = append(keys, k) + } + } + if len(keys) == 0 { + return "; no setting is set for this module" + } + sort.Strings(keys) + return "; set today: " + strings.Join(keys, ", ") +} + +// settingKeysUsedBy is every key a module's files ask for, so a setting that lands in one is not +// called stray. +func settingKeysUsedBy(m Manifest) map[string]bool { + used := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "file" { + continue + } + if content, ok := r["content"].(string); ok { + for _, k := range settingsUsed(content) { + used[k] = true + } + } + } + return used +} diff --git a/internal/catalogue/setting_into_test.go b/internal/catalogue/setting_into_test.go new file mode 100644 index 0000000..4b6f628 --- /dev/null +++ b/internal/catalogue/setting_into_test.go @@ -0,0 +1,55 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// An operator's value reaches a file from the assignment's settings, the node's layer over the +// mesh's (novox/hq ADR 0112, ADR 0155), and a value nothing set is refused by name. +func TestASettingReachesAFileFromTheLayers(t *testing.T) { + file := map[string]any{"id": "env", "type": "file", "path": "/x/mail.env", + "content": "DOMAIN=${setting:domain}\nSITENAME=${setting:sitename}\nWORKERS=${setting:workers}\n"} + layers := []Layer{ + {From: "the mesh", Values: map[string]any{"domain": "example.tld", "sitename": "Mesh", "workers": float64(4)}}, + {From: "this node", Values: map[string]any{"sitename": "This one"}}, + } + if err := settingInto(file, layers, "mail"); err != nil { + t.Fatal(err) + } + if file["content"] != "DOMAIN=example.tld\nSITENAME=This one\nWORKERS=4\n" { + t.Fatalf("filled as %q", file["content"]) + } +} + +func TestASettingNothingSetIsRefusedByName(t *testing.T) { + file := map[string]any{"id": "env", "type": "file", "content": "DOMAIN=${setting:domain}\n"} + err := settingInto(file, []Layer{{From: "the mesh", Values: map[string]any{"other": "x"}}}, "mail") + if err == nil { + t.Fatal("a setting nothing set was written as something") + } + for _, want := range []string{"${setting:domain}", "settings set mail", "set today: other"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal lacks %q: %v", want, err) + } + } + // Left as it was: the literal placeholder must never reach a machine. + if file["content"] != "DOMAIN=${setting:domain}\n" { + t.Fatalf("content was changed on refusal: %q", file["content"]) + } +} + +// A key a file asks for is a destination, so setting it is not called stray. +func TestASettingAFileAsksForIsNotStray(t *testing.T) { + m := Manifest{Module: "mail", Resources: []map[string]any{ + {"id": "env", "type": "file", "content": "DOMAIN=${setting:domain}\n"}, + }} + layers := []Layer{{From: "the mesh", Values: map[string]any{"domain": "example.tld", "stray": "x"}}} + unused := strings.Join(UnusedSettings(m, layers), "; ") + if strings.Contains(unused, `"domain"`) { + t.Fatalf("a key a file asks for was called stray: %s", unused) + } + if !strings.Contains(unused, `"stray"`) { + t.Fatalf("a key nothing reads was not named: %s", unused) + } +} diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index 3dc8d68..77586af 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -173,9 +173,14 @@ func UnusedSettings(m Manifest, layers []Layer) []string { return nil } + // A key a file's content asks for with ${setting:} is a destination too (ADR 0155). + asked := settingKeysUsedBy(m) var unused []string for _, layer := range layers { for key := range layer.Values { + if asked[key] { + continue + } // `expose` is a real destination for a module that listens: it overrides a port's // source (novox/hq ADR 0046), validated in Exposure, so it is not stray here. if key == ExposeSetting && len(m.Listens) > 0 { diff --git a/internal/link/build.go b/internal/link/build.go index ce09fb6..fb02299 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -43,6 +43,12 @@ type BuildRequest struct { // written in a manifest the mesh has not read: it is inside the repository, and reading it is // the build's first act. Held map[string]string `json:"held,omitempty"` + // Seats is the clone base — `scheme://host:port` — of each seat a recipe's context may name + // (novox/hq ADR 0155): `git` for this mesh's own forge. Sent with the asking for the reason + // Held is: the context is written in a manifest the mesh has not read, and only the mesh knows + // which forge holds the seat here. A builder handed no base for a seat a context names refuses + // the build and says so. + Seats map[string]string `json:"seats,omitempty"` } // BuildResult is what a builder says back.