mesh-tools
97a2d841c7
The controller plans config, templates and moved providers' provisions, hands bus accounts to the vault and judges the new manifest words, on the feature's SDK and node-engine validator (feature 57, config and provisioning).
455 lines
17 KiB
Go
455 lines
17 KiB
Go
package catalogue
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1,
|
|
// issue 654).
|
|
//
|
|
// **Two forms, one composition.** A file resource says its text inline (`content`) or names a template file
|
|
// beside its manifest (`"template": "files/<name>"`), never both. The module's build packs `files/` into an
|
|
// archive artifact of its own — an `archive` artifact built `from: "files"` — and resolving the build reads each
|
|
// template's text back from that packed archive into `content` and drops `template`. So the manifest the mesh
|
|
// holds, composition and the node-engine are exactly as they were: a module with its text in `files/` composes
|
|
// byte-identically to the same module with it inline. Where there is no build — `module check`, the merge gate,
|
|
// `module add` — the text is read from the repository tree beside the manifest (WithTemplatesFrom).
|
|
//
|
|
// **files/ unless the manifest names another directory** (issue 731). A module whose files/ is already an archive
|
|
// unpacked onto the machine names its own (`"templates": "templates"`), packed into an archive no resource names, so
|
|
// its templates neither land beside what it unpacks nor change that archive's digest.
|
|
//
|
|
// **Read back by the builder, carried in the resolved manifest.** The builder reads each template from the very
|
|
// bytes it hashed into the archive's digest, so the text the mesh holds is the archive's. The controller
|
|
// registers the resolved manifest as the build result carries it and fetches nothing from the artifact store; a
|
|
// manifest that still names a template when it would be held — built by a builder that predates templates — is
|
|
// refused at registration (HeldTemplateProblem).
|
|
|
|
// TemplateField is the key a file resource names its template file by.
|
|
const TemplateField = "template"
|
|
|
|
// DefaultTemplatesDir is the directory a module's template files are kept in, and its templates archive is built
|
|
// from, when its manifest names none (Manifest.Templates, issue 731).
|
|
const DefaultTemplatesDir = "files"
|
|
|
|
// TemplatesDir is the module's directory its template files are kept in, and the directory its templates archive is
|
|
// built from: the one its manifest names, else files/.
|
|
func (m Manifest) TemplatesDir() string {
|
|
if m.Templates != "" {
|
|
return m.Templates
|
|
}
|
|
return DefaultTemplatesDir
|
|
}
|
|
|
|
// templatesDirProblem is what is wrong with the templates directory a manifest names, or "": one directory of the
|
|
// module's own, a plain relative path inside it — never absolute, never leaving it, never the module's directory
|
|
// itself — and named only by a module that names a template, since a word read by nothing is refused.
|
|
func templatesDirProblem(m Manifest, templated bool) string {
|
|
d := m.Templates
|
|
if d == "" {
|
|
return ""
|
|
}
|
|
if strings.HasPrefix(d, "/") || strings.Contains(d, "\\") || path.Clean(d) != d || d == "." || d == ".." ||
|
|
strings.HasPrefix(d, "../") {
|
|
return fmt.Sprintf("%s names the templates directory %q, which is not a directory of its own: a plain "+
|
|
"relative path inside the module's directory, such as \"templates\" (issue 731)", m.Module, d)
|
|
}
|
|
if !templated {
|
|
return fmt.Sprintf("%s names the templates directory %q and names no template: a file resource names its "+
|
|
"template as %q, or the manifest says no templates directory", m.Module, d, d+"/<name>")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Templated is whether any of the manifest's resources, or any item it contributes (collections.go), names a
|
|
// template.
|
|
func (m Manifest) Templated() bool {
|
|
for _, r := range m.Resources {
|
|
if _, has := r[TemplateField]; has {
|
|
return true
|
|
}
|
|
}
|
|
for _, it := range m.Items {
|
|
if it.Template != "" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// TemplatesArtifact is the name of the archive artifact the module packs its templates into — the one built from its
|
|
// templates directory (TemplatesDir) — or "" when it declares none, or more than one (both refused by templateProblems
|
|
// for a module naming a template).
|
|
func TemplatesArtifact(m Manifest) string {
|
|
found := templatesArtifacts(m)
|
|
if len(found) != 1 {
|
|
return ""
|
|
}
|
|
return found[0]
|
|
}
|
|
|
|
func templatesArtifacts(m Manifest) []string {
|
|
if m.Build == nil {
|
|
return nil
|
|
}
|
|
var found []string
|
|
for _, a := range m.Build.Artifacts {
|
|
if a.Kind == ArtifactArchive && a.URL == "" && cleanFrom(a.From) == m.TemplatesDir() {
|
|
found = append(found, a.Name)
|
|
}
|
|
}
|
|
return found
|
|
}
|
|
|
|
// cleanFrom is an artifact's directory as one relative path: "./files/" is "files".
|
|
func cleanFrom(from string) string {
|
|
c := path.Clean(strings.TrimSpace(from))
|
|
return strings.TrimPrefix(c, "./")
|
|
}
|
|
|
|
// templateOf is the template a resource names, whether it names one, and what is wrong with it ("" when nothing):
|
|
// dir is the module's templates directory.
|
|
func templateOf(module, dir string, r map[string]any) (string, bool, string) {
|
|
raw, has := r[TemplateField]
|
|
if !has {
|
|
return "", false, ""
|
|
}
|
|
id := fmt.Sprint(r["id"])
|
|
if t := fmt.Sprint(r["type"]); t != "file" {
|
|
return "", true, fmt.Sprintf("%s: %s is a %s and names a template; only a file takes its text from one "+
|
|
"(novox/hq ADR 0349 §2)", module, id, t)
|
|
}
|
|
name, ok := raw.(string)
|
|
if !ok || strings.TrimSpace(name) == "" {
|
|
return "", true, fmt.Sprintf("%s: %s names a template that is not a path: %v; a template is %q",
|
|
module, id, raw, dir+"/<name>")
|
|
}
|
|
if _, both := r["content"]; both {
|
|
return name, true, fmt.Sprintf("%s: %s names the template %q and says its content too; a file's text is "+
|
|
"in one place, the template or the content, never both (novox/hq ADR 0349 §2)", module, id, name)
|
|
}
|
|
if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") ||
|
|
strings.Contains(name, "\\") {
|
|
return name, true, fmt.Sprintf("%s: %s names the template %q, which is not a file under %s/ of its own "+
|
|
"repository: a template is %q, a plain relative path, and the build packs only %s/",
|
|
module, id, name, dir, dir+"/<name>", dir)
|
|
}
|
|
return name, true, ""
|
|
}
|
|
|
|
// templateProblems is what is wrong with the manifest's templates, as written: a template on something not a file,
|
|
// beside content, or outside the module's templates directory; a templates directory that is not one, or that no
|
|
// template is in; or a module naming templates whose build packs no archive from that directory. Whether each
|
|
// template exists is judged where a tree is (ReadTemplates) or the archive's text is (Resolve).
|
|
func templateProblems(m Manifest) []string {
|
|
var problems []string
|
|
templated := m.Templated()
|
|
if wrong := templatesDirProblem(m, templated); wrong != "" {
|
|
return append(problems, wrong)
|
|
}
|
|
dir := m.TemplatesDir()
|
|
for _, r := range m.Resources {
|
|
if _, _, wrong := templateOf(m.Module, dir, r); wrong != "" {
|
|
problems = append(problems, wrong)
|
|
}
|
|
}
|
|
if !templated {
|
|
return problems
|
|
}
|
|
switch found := templatesArtifacts(m); len(found) {
|
|
case 0:
|
|
problems = append(problems, fmt.Sprintf("%s names templates and its build packs no archive from %s/: "+
|
|
"declare an artifact {\"name\": %q, \"kind\": \"archive\", \"from\": %q} (novox/hq ADR 0349 §2)",
|
|
m.Module, dir, path.Base(dir), dir))
|
|
case 1:
|
|
default:
|
|
sort.Strings(found)
|
|
problems = append(problems, fmt.Sprintf("%s packs %s/ into %s; its templates are read from one archive",
|
|
m.Module, dir, strings.Join(found, " and ")))
|
|
}
|
|
return problems
|
|
}
|
|
|
|
// fillTemplates is resources with each template's text as its content and the template key dropped: text says the text
|
|
// of a template by its path ("<templates directory>/<name>"), and whether it has one. A template it has no text for, or
|
|
// text that is not UTF-8, is refused by name. Every resource is copied; the manifest's own are left as they were.
|
|
func fillTemplates(module, dir string, resources []map[string]any, text func(string) (string, bool),
|
|
where string) ([]map[string]any, error) {
|
|
out := make([]map[string]any, 0, len(resources))
|
|
var problems []string
|
|
for _, r := range resources {
|
|
name, has, wrong := templateOf(module, dir, r)
|
|
if !has {
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
if wrong != "" {
|
|
problems = append(problems, wrong)
|
|
continue
|
|
}
|
|
body, ok := text(name)
|
|
if !ok {
|
|
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and %s holds no such file",
|
|
module, r["id"], name, where))
|
|
continue
|
|
}
|
|
if !utf8.ValidString(body) {
|
|
// A manifest is JSON: text that is not UTF-8 would be changed on the way, silently.
|
|
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not UTF-8 text; a "+
|
|
"file's text is carried as JSON and would not reach the machine as written", module, r["id"], name))
|
|
continue
|
|
}
|
|
filled := make(map[string]any, len(r))
|
|
for k, v := range r {
|
|
filled[k] = v
|
|
}
|
|
delete(filled, TemplateField)
|
|
filled["content"] = body
|
|
out = append(out, filled)
|
|
}
|
|
if len(problems) > 0 {
|
|
return nil, errors.New(strings.Join(problems, "; "))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// fillItemTemplates is items with each template's text as its content and the template dropped, as fillTemplates
|
|
// does for files (collections.go).
|
|
func fillItemTemplates(m Manifest, items []Item, text func(string) (string, bool), where string) ([]Item, error) {
|
|
if len(items) == 0 {
|
|
return items, nil
|
|
}
|
|
out := make([]Item, 0, len(items))
|
|
var problems []string
|
|
for _, it := range items {
|
|
if it.Template == "" {
|
|
out = append(out, it)
|
|
continue
|
|
}
|
|
if wrong := itemTemplateProblem(m, it.Template); wrong != "" {
|
|
problems = append(problems, fmt.Sprintf("%s's item of %s: %s", m.Module, it.Collection, wrong))
|
|
continue
|
|
}
|
|
body, ok := text(it.Template)
|
|
switch {
|
|
case !ok:
|
|
problems = append(problems, fmt.Sprintf("%s: its item of %s names the template %q, and %s holds no "+
|
|
"such file", m.Module, it.Collection, it.Template, where))
|
|
continue
|
|
case !utf8.ValidString(body):
|
|
problems = append(problems, fmt.Sprintf("%s: its item of %s names the template %q, which is not UTF-8 "+
|
|
"text", m.Module, it.Collection, it.Template))
|
|
continue
|
|
}
|
|
it.Template, it.Content = "", body
|
|
out = append(out, it)
|
|
}
|
|
if len(problems) > 0 {
|
|
return nil, errors.New(strings.Join(problems, "; "))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository tree,
|
|
// by template path. Refused by name: a template the directory lacks, and anything under its templates directory
|
|
// (TemplatesDir) the build's packing refuses — that directory itself, or any directory or file under it, that is a
|
|
// symlink or not a plain file — so nothing judged here can be read from outside the tree, and no tree passes here that
|
|
// the build would refuse. Each template is read through an os.Root of the module's directory besides, which no path
|
|
// element can leave.
|
|
func ReadTemplates(m Manifest, dir string) (map[string]string, error) {
|
|
problems := templatesTreeProblems(m.Module, dir, m.TemplatesDir())
|
|
root, err := os.OpenRoot(dir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: the module's directory cannot be opened to read its templates: %w", m.Module, err)
|
|
}
|
|
defer root.Close()
|
|
texts := map[string]string{}
|
|
type named struct {
|
|
id any
|
|
template string
|
|
wrong string
|
|
}
|
|
var templates []named
|
|
for _, r := range m.Resources {
|
|
name, has, wrong := templateOf(m.Module, m.TemplatesDir(), r)
|
|
if has {
|
|
templates = append(templates, named{r["id"], name, wrong})
|
|
}
|
|
}
|
|
for _, it := range m.Items {
|
|
if it.Template != "" {
|
|
templates = append(templates, named{"its item of " + it.key(), it.Template,
|
|
itemTemplateProblem(m, it.Template)})
|
|
}
|
|
}
|
|
for _, t := range templates {
|
|
r := map[string]any{"id": t.id}
|
|
name, wrong := t.template, t.wrong
|
|
if wrong != "" {
|
|
problems = append(problems, wrong)
|
|
continue
|
|
}
|
|
if _, read := texts[name]; read {
|
|
continue
|
|
}
|
|
info, err := root.Lstat(name)
|
|
switch {
|
|
case errors.Is(err, fs.ErrNotExist):
|
|
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+
|
|
"has no such file", m.Module, r["id"], name))
|
|
continue
|
|
case err != nil:
|
|
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v",
|
|
m.Module, r["id"], name, err))
|
|
continue
|
|
case !info.Mode().IsRegular():
|
|
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not a plain file; "+
|
|
"the build packs only files", m.Module, r["id"], name))
|
|
continue
|
|
}
|
|
body, err := root.ReadFile(name)
|
|
if err != nil {
|
|
problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v",
|
|
m.Module, r["id"], name, err))
|
|
continue
|
|
}
|
|
texts[name] = string(body)
|
|
}
|
|
if len(problems) > 0 {
|
|
return nil, errors.New(strings.Join(problems, "; "))
|
|
}
|
|
return texts, nil
|
|
}
|
|
|
|
// templatesTreeProblems is everything under the module's templates directory (templates, inside dir) that the
|
|
// build's packing refuses: that directory itself not a plain directory, and any entry under it that is neither a
|
|
// directory nor a plain file — a symlink, to a file or a directory, a device, a pipe. Each said by its path. A
|
|
// templates directory that does not exist is no problem here: each template it would have held is refused as missing.
|
|
func templatesTreeProblems(module, dir, templates string) []string {
|
|
top := filepath.Join(dir, filepath.FromSlash(templates))
|
|
info, err := os.Lstat(top)
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("%s: %s/ cannot be read: %v", module, templates, err)}
|
|
}
|
|
if !info.IsDir() {
|
|
return []string{fmt.Sprintf("%s: %s/ is %s, not a directory of the module's own; its templates are read "+
|
|
"only from inside the repository, and the build packs only directories and files", module, templates,
|
|
kindOf(info.Mode()))}
|
|
}
|
|
var problems []string
|
|
err = filepath.WalkDir(top, func(at string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() || d.Type().IsRegular() {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(dir, at)
|
|
problems = append(problems, fmt.Sprintf("%s: %s is %s; the build packs only directories and files, so "+
|
|
"%s/ holds nothing else", module, filepath.ToSlash(rel), kindOf(d.Type()), templates))
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
problems = append(problems, fmt.Sprintf("%s: %s/ cannot be read: %v", module, templates, err))
|
|
}
|
|
return problems
|
|
}
|
|
|
|
// kindOf says what a file that is not a plain file is.
|
|
func kindOf(mode fs.FileMode) string {
|
|
switch {
|
|
case mode&fs.ModeSymlink != 0:
|
|
return "a symlink"
|
|
case mode&fs.ModeDevice != 0:
|
|
return "a device"
|
|
case mode&fs.ModeNamedPipe != 0:
|
|
return "a named pipe"
|
|
case mode&fs.ModeSocket != 0:
|
|
return "a socket"
|
|
case mode.IsDir():
|
|
return "a directory"
|
|
}
|
|
return "not a plain file"
|
|
}
|
|
|
|
// WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository
|
|
// tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read
|
|
// again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges
|
|
// inline content. A manifest naming no template is returned as it is.
|
|
func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) {
|
|
if !m.Templated() {
|
|
return m, nil
|
|
}
|
|
texts, err := ReadTemplates(m, dir)
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
filled, err := fillTemplates(m.Module, m.TemplatesDir(), m.Resources, func(name string) (string, bool) {
|
|
t, ok := texts[name]
|
|
return t, ok
|
|
}, "the module's directory")
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
items, err := fillItemTemplates(m, m.Items, func(name string) (string, bool) {
|
|
t, ok := texts[name]
|
|
return t, ok
|
|
}, "the module's directory")
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
out := m
|
|
out.Resources = filled
|
|
out.Items = items
|
|
// Nothing names a template now, so nothing names its directory.
|
|
out.Templates = ""
|
|
// A moved module's mark was judged above, while it still named its templates; read again with its text inline,
|
|
// it is judged as it composes without its build's templates archive — as before, nothing handed (handed.go).
|
|
out.Moved = false
|
|
raw, err := json.Marshal(out)
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
again, err := parseManifest(raw, m.predefined)
|
|
if err != nil {
|
|
return Manifest{}, err
|
|
}
|
|
again.Moved = m.Moved
|
|
return again, nil
|
|
}
|
|
|
|
// HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the
|
|
// manifest the mesh holds carries every file's text, which a build reads back from its templates archive. One
|
|
// naming a template was resolved by a builder that does not read templates, and would compose files with no text.
|
|
func HeldTemplateProblem(m Manifest) string {
|
|
var named []string
|
|
for _, r := range m.Resources {
|
|
if t, has := r[TemplateField]; has {
|
|
named = append(named, fmt.Sprintf("%v (%v)", r["id"], t))
|
|
}
|
|
}
|
|
for _, it := range m.Items {
|
|
if it.Template != "" {
|
|
named = append(named, fmt.Sprintf("its item of %s (%s)", it.Collection, it.Template))
|
|
}
|
|
}
|
|
if len(named) == 0 {
|
|
return ""
|
|
}
|
|
return fmt.Sprintf("%s still names the template(s) of %s: the manifest the mesh holds carries each file's "+
|
|
"text, read back from the build's templates archive — this one was resolved by a builder that does not "+
|
|
"read templates, or not built at all (novox/hq ADR 0349 §2)", m.Module, strings.Join(named, ", "))
|
|
}
|