diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index e20b61a2..17572a38 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -37,6 +37,11 @@ type Layer struct { // Default marks the layer a module's own defaults make (novox/hq ADR 0262). Marked rather than // recognised by From, which is a node's name for a node's layer, and a node may be called anything. Default bool + // Placement is the assignment's placement at this layer's scope — `places`, `accesses`, `networks` + // — which the controller holds apart from settings (novox/hq ADR 0348 clause 11, issue 651): read + // by Places, AccessPlaces and KeptNetworks, and never from Values. LayerOf splits a layer as a person + // says it into the two. + Placement map[string]any } // ApplySettings produces a resource's final content from the module's own and the layers over it. @@ -45,6 +50,10 @@ type Layer struct { // module first, then this node's. A node that differs from the rest is expressed by differing, // rather than by repeating everything the rest already say. func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, error) { + // Today's form reaches a mergeable file with every key of the layer as it was said, placement too + // (novox/hq ADR 0326 §3, ADR 0348 clause 11: a module not yet moved keeps it), so a file composes + // byte for byte as it did while placement was kept in the layer. + layers = asSaid(layers) how, _ := resource["merge"].(string) taken, names := takenKeys(resource) if how == "" { @@ -108,25 +117,27 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err return out, nil } -// Settle lays settings over what a provider serves. Exported because a served fact is settled where -// the mesh is walked rather than where a node is declared. +// Settle fills what a provider serves from its settings. Exported because a served fact is settled +// where the mesh is walked rather than where a node is declared. // -// **A setting overrides a served key; it never adds one** (novox/hq 04-ISSUES/173). What a consumer -// is told is the provider's contract, and a setting made for one of the provider's files — a site -// name, a public address — is not part of it. Before this, every setting of a module reached every -// consumer of every provision it served. +// **What a provider serves is its definition's; a layer replaces none of it** (novox/hq ADR 0348 +// clause 11, issue 696). A served value is filled from the layers only where the definition asks, +// with `${setting:}` (ADR 0155) — a mail provider's domain, an identity provider's issuer, a +// time-series store's org and bucket — and a layer that sets a key the provider serves as a literal +// changes nothing a consumer is told. Nor does a setting add a key (novox/hq 04-ISSUES/173): a +// setting made for one of the provider's files — a site name, a public address — is not part of the +// contract. Until issue 696, a layer replaced any served key it named; influxdb on ace was the one +// layer relying on it, and serves `${setting:org}` and `${setting:bucket}` since. func Settle(base map[string]any, layers []Layer) (map[string]any, error) { - return overridden(base, layers, "what is served") + return filledFromSettings(deepCopy(base), layers, "what is served") } -// overridden lays settings over a map whose keys are its contract: a contribution, a served fact. -// Only the keys the map already declares are touched; the rest of a layer is somebody else's -// business (a file's, another destination's) and is left to reach it there. +// overridden lays settings over a contribution: a map whose keys are its contract. Only the keys the +// map already declares are touched; the rest of a layer is somebody else's business (a file's, +// another destination's) and is left to reach it there. // -// A declared value may itself be the operator's, `${setting:}` (ADR 0155): a mail provider -// serves its domain, an identity provider its issuer, and neither is the definition's to state. -// Filled from the layers after the overrides, and refused by name when nothing sets it — a literal -// placeholder handed to a consumer is a service configured against a string nobody meant. +// A declared value may itself be the operator's, `${setting:}` (ADR 0155), filled after the +// overrides (filledFromSettings). func overridden(base map[string]any, layers []Layer, what string) (map[string]any, error) { kept := make([]Layer, 0, len(layers)) for _, layer := range layers { @@ -142,6 +153,14 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an if err != nil { return nil, err } + return filledFromSettings(merged, layers, what) +} + +// filledFromSettings fills each `${setting:}` in a map's top-level strings from the layers, +// later beating earlier and a module's defaults lowest, and refuses one nothing sets by name — a +// literal placeholder handed to a consumer is a service configured against a string nobody meant. +// It fills merged in place and returns it. +func filledFromSettings(merged map[string]any, layers []Layer, what string) (map[string]any, error) { for key, value := range merged { s, ok := value.(string) if !ok { @@ -168,7 +187,8 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an // module says what it means by default, and somebody says what it means here. A contribution that // could not be settled would have to be edited to be reused anywhere else. The two differ in one // respect, and the caller decides it: a file takes keys it did not declare (a setting may add to a -// configuration), a contribution or served fact does not (overridden). +// configuration), a contribution does not (overridden). A served fact takes no layer at all; only +// its ${setting:} is filled (Settle). func settle(base map[string]any, layers []Layer, protected map[string]bool, what string) ( map[string]any, error) { merged := deepCopy(base) @@ -344,8 +364,9 @@ func sortedKeysAny(m map[string]any) []string { // way there is. This is what makes that visible at the moment they set it. // // Where a key can land: any mergeable file takes any key; a file asking for `${setting:}` -// takes that key (ADR 0155); a contribution or a served fact takes a key it declares, and no other -// (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` — +// takes that key (ADR 0155); a contribution takes a key it declares, and no other (novox/hq +// 04-ISSUES/173); a served fact takes only the keys it asks for as `${setting:}`, since a layer +// replaces no served key (novox/hq issue 696); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` — // are read by the mesh. A key none of those takes is stray, and is said so rather than dropped. func UnusedSettings(m Manifest, layers []Layer) []string { named := namesItsKeys(m) @@ -388,14 +409,9 @@ func UnusedSettings(m Manifest, layers []Layer) []string { } } } - for _, served := range m.Serves { - for key := range served { - lands[key] = true - } - } var unused []string for _, layer := range layers { - for key := range layer.Values { + for key := range layer.Whole() { if lands[key] { continue } @@ -435,7 +451,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string { } unused = append(unused, fmt.Sprintf( "%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+ - "declares no %q in what it contributes or serves%s", + "declares no %q in what it contributes%s", layer.From, key, m.Module, key, key, takenSaid(m))) } } @@ -483,7 +499,7 @@ func KeptNetworks(m Manifest, layers []Layer, adopted bool) (map[string][]string } out := map[string][]string{} for _, layer := range layers { - raw, ok := layer.Values[NetworksSetting] + raw, ok := layer.Placement[NetworksSetting] if !ok { continue } @@ -543,7 +559,8 @@ var networkName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) // refused where it is stored (SetSettings, with UnusedSettings) and said where a plan is read, // and never costs a module its place. func JudgeSettings(m Manifest, layers []Layer, adopted bool) error { - if err := settingsHoldOneLine(m.Module, layers); err != nil { + // Placement is judged for line ends as it always was: a path is written into files as well. + if err := settingsHoldOneLine(m.Module, asSaid(layers)); err != nil { return err } // Which keys each mergeable file takes is the definition's to say right (novox/hq ADR 0326); @@ -580,7 +597,7 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error { for k, v := range settled { copied[k] = v } - if err := settingInto(copied, WithDefaults(m, layers), m.Module); err != nil { + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module), m); err != nil { return err } } diff --git a/internal/catalogue/settings_judge_test.go b/internal/catalogue/settings_judge_test.go index 50d96222..cbb494c9 100644 --- a/internal/catalogue/settings_judge_test.go +++ b/internal/catalogue/settings_judge_test.go @@ -27,7 +27,7 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te if c.name == "a mesh-wide port" { from = MeshWideLayer } - err := JudgeSettings(web, []Layer{{From: from, Values: c.layer}}, true) + err := JudgeSettings(web, []Layer{LayerOf(from, c.layer)}, true) if err == nil || !strings.Contains(err.Error(), c.refuse) { t.Errorf("%s: judged %v, want %q", c.name, err, c.refuse) } @@ -69,8 +69,8 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te // declaration as the networks it also joins. func TestAKeptNetworkReachesTheContainerOnAnAdoptedMachineOnly(t *testing.T) { r := anAdoptedAnchor() - keep := SettingsBy{"hello-web": {{From: "anchor", - Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}}}}} + keep := SettingsBy{"hello-web": {LayerOf("anchor", + map[string]any{NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}})}} with := anchorRendering(true) with.Settings = keep @@ -107,13 +107,13 @@ func TestAKeptNetworkReachesTheContainerOnAnAdoptedMachineOnly(t *testing.T) { layer Layer want string }{ - {"mesh-wide", Layer{From: MeshWideLayer, Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"x"}}}}, + {"mesh-wide", LayerOf(MeshWideLayer, map[string]any{NetworksSetting: map[string]any{"server": []any{"x"}}}), "a found network is a fact about one machine"}, - {"an unknown container", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"db": []any{"x"}}}}, + {"an unknown container", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"db": []any{"x"}}}), `names the container "db", which it does not declare`}, - {"not a list", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"server": "x"}}}, + {"not a list", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"server": "x"}}), "is a list of network names"}, - {"not a network name", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"a/b"}}}}, + {"not a network name", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"server": []any{"a/b"}}}), "which is not a network name"}, } { _, err := KeptNetworks(web, []Layer{c.layer}, true) diff --git a/internal/catalogue/settings_test.go b/internal/catalogue/settings_test.go index 6fe6d2a0..2313d8be 100644 --- a/internal/catalogue/settings_test.go +++ b/internal/catalogue/settings_test.go @@ -190,22 +190,6 @@ func TestASettingLandsOnlyWhereSomethingDeclaresIt(t *testing.T) { } } -func TestASettingOverridesAServedKeyAndAddsNone(t *testing.T) { - // What a consumer is told is the provider's contract. A setting made for one of the - // provider's files — its site name, its public address — is not part of it. - served, err := Settle(map[string]any{"host": "mail", "port": 25}, - []Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}}) - if err != nil { - t.Fatal(err) - } - if served["host"] != "post" { - t.Errorf("the setting did not override the served host: %v", served) - } - if _, leaked := served["sitename"]; leaked { - t.Errorf("a setting for a file reached the consumers: %v", served) - } -} - func TestContentThatIsNotJSONIsRefusedWhereSomebodyIsLooking(t *testing.T) { // Rather than on the machine, at apply time, as a file the program cannot read. _, err := ApplySettings(file(`this is not json`), nil) diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go new file mode 100644 index 00000000..2a363bf2 --- /dev/null +++ b/internal/catalogue/template.go @@ -0,0 +1,334 @@ +package catalogue + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "unicode/utf8" +) + +// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, +// issue 654). +// +// **Two forms, one composition.** A file resource says its text inline (`content`) or names a template file +// beside its manifest (`"template": "files/"`), never both. The module's build packs `files/` into an +// archive artifact of its own — an `archive` artifact built `from: "files"` — and resolving the build reads each +// template's text back from that packed archive into `content` and drops `template`. So the manifest the mesh +// holds, composition and the node-engine are exactly as they were: a module with its text in `files/` composes +// byte-identically to the same module with it inline. Where there is no build — `module check`, the merge gate, +// `module add` — the text is read from the repository tree beside the manifest (WithTemplatesFrom). +// +// **Read back by the builder, carried in the resolved manifest.** The builder reads each template from the very +// bytes it hashed into the archive's digest, so the text the mesh holds is the archive's. The controller +// registers the resolved manifest as the build result carries it and fetches nothing from the artifact store; a +// manifest that still names a template when it would be held — built by a builder that predates templates — is +// refused at registration (HeldTemplateProblem). + +// TemplateField is the key a file resource names its template file by. +const TemplateField = "template" + +// TemplatesDir is the module's directory its template files are kept in, and the directory its templates archive +// is built from. +const TemplatesDir = "files" + +// Templated is whether any of the manifest's resources names a template. +func (m Manifest) Templated() bool { + for _, r := range m.Resources { + if _, has := r[TemplateField]; has { + return true + } + } + return false +} + +// TemplatesArtifact is the name of the archive artifact the module packs its templates into — the one built from +// files/ — or "" when it declares none, or more than one (both refused by templateProblems for a module naming a +// template). +func TemplatesArtifact(m Manifest) string { + found := templatesArtifacts(m) + if len(found) != 1 { + return "" + } + return found[0] +} + +func templatesArtifacts(m Manifest) []string { + if m.Build == nil { + return nil + } + var found []string + for _, a := range m.Build.Artifacts { + if a.Kind == ArtifactArchive && a.URL == "" && cleanFrom(a.From) == TemplatesDir { + found = append(found, a.Name) + } + } + return found +} + +// cleanFrom is an artifact's directory as one relative path: "./files/" is "files". +func cleanFrom(from string) string { + c := path.Clean(strings.TrimSpace(from)) + return strings.TrimPrefix(c, "./") +} + +// templateOf is the template a resource names, whether it names one, and what is wrong with it ("" when nothing). +func templateOf(module string, r map[string]any) (string, bool, string) { + raw, has := r[TemplateField] + if !has { + return "", false, "" + } + id := fmt.Sprint(r["id"]) + if t := fmt.Sprint(r["type"]); t != "file" { + return "", true, fmt.Sprintf("%s: %s is a %s and names a template; only a file takes its text from one "+ + "(novox/hq ADR 0349 §2)", module, id, t) + } + name, ok := raw.(string) + if !ok || strings.TrimSpace(name) == "" { + return "", true, fmt.Sprintf("%s: %s names a template that is not a path: %v; a template is %q", + module, id, raw, TemplatesDir+"/") + } + if _, both := r["content"]; both { + return name, true, fmt.Sprintf("%s: %s names the template %q and says its content too; a file's text is "+ + "in one place, the template or the content, never both (novox/hq ADR 0349 §2)", module, id, name) + } + if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, TemplatesDir+"/") || + strings.Contains(name, "\\") { + return name, true, fmt.Sprintf("%s: %s names the template %q, which is not a file under %s/ of its own "+ + "repository: a template is %q, a plain relative path, and the build packs only %s/", + module, id, name, TemplatesDir, TemplatesDir+"/", TemplatesDir) + } + return name, true, "" +} + +// templateProblems is what is wrong with the manifest's templates, as written: a template on something not a file, +// beside content, outside files/, or a module naming templates whose build packs no archive from files/. Whether +// each template exists is judged where a tree is (ReadTemplates) or the archive's text is (Resolve). +func templateProblems(m Manifest) []string { + var problems []string + templated := false + for _, r := range m.Resources { + _, has, wrong := templateOf(m.Module, r) + templated = templated || has + if wrong != "" { + problems = append(problems, wrong) + } + } + if !templated { + return problems + } + switch found := templatesArtifacts(m); len(found) { + case 0: + problems = append(problems, fmt.Sprintf("%s names templates and its build packs no archive from %s/: "+ + "declare an artifact {\"name\": \"files\", \"kind\": \"archive\", \"from\": %q} (novox/hq ADR 0349 §2)", + m.Module, TemplatesDir, TemplatesDir)) + case 1: + default: + sort.Strings(found) + problems = append(problems, fmt.Sprintf("%s packs %s/ into %s; its templates are read from one archive", + m.Module, TemplatesDir, strings.Join(found, " and "))) + } + return problems +} + +// fillTemplates is resources with each template's text as its content and the template key dropped: text says the +// text of a template by its path ("files/"), and whether it has one. A template it has no text for, or text +// that is not UTF-8, is refused by name. Every resource is copied; the manifest's own are left as they were. +func fillTemplates(module string, resources []map[string]any, text func(string) (string, bool), + where string) ([]map[string]any, error) { + out := make([]map[string]any, 0, len(resources)) + var problems []string + for _, r := range resources { + name, has, wrong := templateOf(module, r) + if !has { + out = append(out, r) + continue + } + if wrong != "" { + problems = append(problems, wrong) + continue + } + body, ok := text(name) + if !ok { + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and %s holds no such file", + module, r["id"], name, where)) + continue + } + if !utf8.ValidString(body) { + // A manifest is JSON: text that is not UTF-8 would be changed on the way, silently. + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not UTF-8 text; a "+ + "file's text is carried as JSON and would not reach the machine as written", module, r["id"], name)) + continue + } + filled := make(map[string]any, len(r)) + for k, v := range r { + filled[k] = v + } + delete(filled, TemplateField) + filled["content"] = body + out = append(out, filled) + } + if len(problems) > 0 { + return nil, errors.New(strings.Join(problems, "; ")) + } + return out, nil +} + +// ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository +// tree, by template path. Refused by name: a template the directory lacks, and anything under files/ the build's +// packing refuses — files/ itself, or any directory or file under it, that is a symlink or not a plain file — so +// nothing judged here can be read from outside the tree, and no tree passes here that the build would refuse. Each +// template is read through an os.Root of the module's directory besides, which no path element can leave. +func ReadTemplates(m Manifest, dir string) (map[string]string, error) { + problems := templatesTreeProblems(m.Module, dir) + root, err := os.OpenRoot(dir) + if err != nil { + return nil, fmt.Errorf("%s: the module's directory cannot be opened to read its templates: %w", m.Module, err) + } + defer root.Close() + texts := map[string]string{} + for _, r := range m.Resources { + name, has, wrong := templateOf(m.Module, r) + if !has { + continue + } + if wrong != "" { + problems = append(problems, wrong) + continue + } + if _, read := texts[name]; read { + continue + } + info, err := root.Lstat(name) + switch { + case errors.Is(err, fs.ErrNotExist): + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+ + "has no such file", m.Module, r["id"], name)) + continue + case err != nil: + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", + m.Module, r["id"], name, err)) + continue + case !info.Mode().IsRegular(): + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not a plain file; "+ + "the build packs only files", m.Module, r["id"], name)) + continue + } + body, err := root.ReadFile(name) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", + m.Module, r["id"], name, err)) + continue + } + texts[name] = string(body) + } + if len(problems) > 0 { + return nil, errors.New(strings.Join(problems, "; ")) + } + return texts, nil +} + +// templatesTreeProblems is everything under the module's files/ that the build's packing refuses: files/ itself not +// a plain directory, and any entry under it that is neither a directory nor a plain file — a symlink, to a file or a +// directory, a device, a pipe. Each said by its path. A files/ that does not exist is no problem here: each template +// it would have held is refused as missing. +func templatesTreeProblems(module, dir string) []string { + top := filepath.Join(dir, TemplatesDir) + info, err := os.Lstat(top) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return []string{fmt.Sprintf("%s: %s/ cannot be read: %v", module, TemplatesDir, err)} + } + if !info.IsDir() { + return []string{fmt.Sprintf("%s: %s/ is %s, not a directory of the module's own; its templates are read "+ + "only from inside the repository, and the build packs only directories and files", module, TemplatesDir, + kindOf(info.Mode()))} + } + var problems []string + err = filepath.WalkDir(top, func(at string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Type().IsRegular() { + return nil + } + rel, _ := filepath.Rel(dir, at) + problems = append(problems, fmt.Sprintf("%s: %s is %s; the build packs only directories and files, so "+ + "%s/ holds nothing else", module, filepath.ToSlash(rel), kindOf(d.Type()), TemplatesDir)) + return nil + }) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %s/ cannot be read: %v", module, TemplatesDir, err)) + } + return problems +} + +// kindOf says what a file that is not a plain file is. +func kindOf(mode fs.FileMode) string { + switch { + case mode&fs.ModeSymlink != 0: + return "a symlink" + case mode&fs.ModeDevice != 0: + return "a device" + case mode&fs.ModeNamedPipe != 0: + return "a named pipe" + case mode&fs.ModeSocket != 0: + return "a socket" + case mode.IsDir(): + return "a directory" + } + return "not a plain file" +} + +// WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository +// tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read +// again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges +// inline content. A manifest naming no template is returned as it is. +func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) { + if !m.Templated() { + return m, nil + } + texts, err := ReadTemplates(m, dir) + if err != nil { + return Manifest{}, err + } + filled, err := fillTemplates(m.Module, m.Resources, func(name string) (string, bool) { + t, ok := texts[name] + return t, ok + }, "the module's directory") + if err != nil { + return Manifest{}, err + } + out := m + out.Resources = filled + raw, err := json.Marshal(out) + if err != nil { + return Manifest{}, err + } + return ParseManifest(raw) +} + +// HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the +// manifest the mesh holds carries every file's text, which a build reads back from its templates archive. One +// naming a template was resolved by a builder that does not read templates, and would compose files with no text. +func HeldTemplateProblem(m Manifest) string { + var named []string + for _, r := range m.Resources { + if t, has := r[TemplateField]; has { + named = append(named, fmt.Sprintf("%v (%v)", r["id"], t)) + } + } + if len(named) == 0 { + return "" + } + return fmt.Sprintf("%s still names the template(s) of %s: the manifest the mesh holds carries each file's "+ + "text, read back from the build's templates archive — this one was resolved by a builder that does not "+ + "read templates, or not built at all (novox/hq ADR 0349 §2)", m.Module, strings.Join(named, ", ")) +}