package catalogue import ( "fmt" "path/filepath" "regexp" "sort" "strings" ) // Where a module's data is on THIS machine is the assignment's (novox/hq ADR 0112, issue 153). // // A definition names no host path. It declares the directories it owns by id, and the operator's // shared data it needs by id too (an `access`, ADR 0051). A node has a default layout for the // former — // — and nothing at all for the latter, because shared data is // wherever the operator keeps it. An adopted machine keeps its data where the predecessor put it: // a 40 TB library on its own pool, a configuration on a second disk. Both halves are said on the // assignment, validated the way `endpoints` is — an id the module does not declare is refused, // because a setting that reaches nothing is a mistake — and resolved here, so the host receives // concrete paths exactly as it always has and learns no field. // // {"places": {"config": "/services/sonarr/config", // "data": {"path": "/mnt/plex/data", "owner": "1000:1000"}}, // "accesses": {"series": "/storage/media/series", // "downloads": "/storage/downloads"}} // // A placed directory is still the mesh's: created, chowned to the owner the assignment says (or // the manifest's), removed when empty and no longer declared. A placed access is still the // operator's: mounted, never created, chowned or removed. // PlacesSetting is the settings key that places a module's declared directories, by id. const PlacesSetting = "places" // AccessesSetting is the settings key that says where a module's accesses are on this node, by id. const AccessesSetting = "accesses" // Placement is what an assignment says about one of a module's directories. type Placement struct { // Path is where the directory is on this machine. Absolute. Path string // Owner is "uid:gid" when the assignment overrides the manifest's — the predecessor's data is // owned by whoever it ran as, and that is one machine's fact. Owner string } var ownerShape = regexp.MustCompile(`^[0-9]+:[0-9]+$`) // Where no placement and no access may be, from any route, the controller's terminal too (novox/hq issue 339). // // A placed directory is created and owned by the node-engine as root, with the owner the setting names, and // whatever the module writes into it is written as root; an access is mounted into the module's container, // which may run as root. A place at /etc owned by an account a caller names hands that account the machine, // and an access at / mounts the machine's root into a container. So the machine's own trees are refused here, // before anything is kept or composed, and the node-engine refuses them again where it applies. // // systemTrees are refused at and below: the machine's system, the kernel's, the boot loader's, root's home, // what lives only while the machine runs, the spool (cron's tables are there), the container runtimes' data // (every container's filesystem), /opt, and the node-engine's and the mesh's own state. Any home's .ssh is // refused as well, wherever the home is. systemRoots are // refused at, and wherever a path holds one (an ancestor of /var/lib holds it): each is the parent of every // module's or every person's directories, and owning it is owning all of them. var ( systemTrees = []string{"/etc", "/usr", "/boot", "/root", "/run", "/var/run", "/var/lock", "/proc", "/sys", "/dev", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/var/lib/mesh", "/var/lib/mesh-host", "/var/spool", "/var/lib/docker", "/var/lib/containers", "/var/lib/containerd", "/opt"} systemRoots = []string{"/", "/var", "/var/lib", "/var/cache", "/var/log", "/var/tmp", "/home", "/mnt", "/media", "/srv", "/tmp", "/storage", "/data", "/services"} ) // systemPath says why a clean absolute path is the machine's own and never a placement's or an access's, or "". func systemPath(path string) string { // Any home's keys, wherever the home is: a .ssh directory is its account's, and the keys and the list of who // may log in as it are in there. for _, part := range strings.Split(path, "/") { if part == ".ssh" { return path + " is an account's .ssh, which holds its keys and who may log in as it" } } for _, tree := range systemTrees { if path == tree || strings.HasPrefix(path, tree+"/") { return path + " is in " + tree + ", the machine's own or the mesh's state" } if strings.HasPrefix(tree, path+"/") || path == "/" { return path + " holds " + tree + ", the machine's own or the mesh's state" } } for _, root := range systemRoots { if path == root { return path + " is the parent of every module's or every person's directories" } } return "" } // Where no directory or file a module's definition resolves to may be (novox/hq issue 496). // // mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to /docker: with the // default root, /var/lib/docker, every container's filesystem. systemPath judged only the `places` and `accesses` // settings, so the default layout and a definition's own paths reached the merge gate unjudged; it composed every // machine and passed, and only the node-engine refused the directory, at apply, failing the walk. // // **Judged where a definition is judged, never where a machine is composed.** resolvedPathProblems runs in // ParseManifest, which every route a definition takes into the mesh passes: `module check`, registration of a // build (the builder's and the registry verbs'), and the merge gate's reading of the changed repository. A // refusal there stops one definition before it reaches any machine. Composition reads registered manifests // without ParseManifest, and judges nothing of this: refusing there would fail the whole machine's declaration and // freeze every module on it for one module's path, where the node-engine fails only that resource. // // **The node-engine's rules, no more** (mesh-host's internal/apply/placement_guard.go, with files judged as // directories are after novox/hq issue 495, rule 6). A path is refused when it is one of protectedRoots or holds // one, when it is at or below a tree in forbiddenBelow, or at or below one of engineTrees and its module is not // the node-engine's. What the engine judges with what only the machine knows stays the engine's: where the // runtimes really keep their data, links, the accounts' homes, a directory's owner below /etc. The lists are the // engine's own words, and a test (engine_guard_test.go) holds them equal to mesh-host's beside this repository. // systemPath stays the stricter rule for a setting: a setting is an operator's word about one machine, and the // trees it lists (/etc, /usr, /run, the mesh's own) are where the mesh's own modules write by design. var ( protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64", "/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin", "/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin", "/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp", "/var/spool"} forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker", "/var/lib/containers", "/var/lib/containerd"} engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"} ) // engineModule is the node-engine's own module, the one that places in engineTrees. const engineModule = "mesh-host" // enginePath says why the node-engine refuses a directory or file at path for module, or "". func enginePath(path, module string) string { path = filepath.Clean(path) if !filepath.IsAbs(path) { return "" } atOrBelow := func(tree string) bool { return path == tree || strings.HasPrefix(path, tree+"/") } for _, root := range protectedRoots { if path == root || path == "/" || strings.HasPrefix(root, path+"/") { return root + " is one of the machine's own directories, and owning it is owning everything in it" } } for _, tree := range forbiddenBelow { if atOrBelow(tree) { return "nothing is placed in " + tree } } if module != engineModule { for _, tree := range engineTrees { if atOrBelow(tree) { return tree + " is the node-engine's own, placed in by its own module alone" } } } return "" } // resolvedPathProblems is every directory and file of the definition whose path, resolved as a node with the // default root resolves it, the node-engine would refuse (novox/hq issue 496). A path still holding a placeholder // only a machine fills (a setting, an access the definition gives no default) is the engine's to judge. func (m Manifest) resolvedPathProblems() []string { dirs := dirsFor(m, Rendering{}) accesses := map[string]string{} for _, a := range m.Accesses { if a.ID != "" && a.Path != "" { accesses[a.ID] = a.Path } } var problems []string for _, r := range m.Resources { kind := fmt.Sprint(r["type"]) if kind != "directory" && kind != "file" { continue } id := fmt.Sprint(r["id"]) path, _ := r["path"].(string) if kind == "directory" && path == "" { path = dirs[id] } path, _ = dirFill(path, dirs, m.Module) path, _ = accessFill(path, accesses, m.Module) if path == "" || strings.Contains(path, "${") { continue } if why := enginePath(path, m.Module); why != "" { problems = append(problems, fmt.Sprintf("%s's %s %q resolves to %s, which the node-engine refuses: %s. "+ "A module's directories and files are judged when its definition is, so the merge gate refuses it "+ "before any machine does (novox/hq issue 496)", m.Module, kind, id, filepath.Clean(path), why)) } } return problems } // accessRef is how a module names one of its accesses: ${access:}. var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`) // Places reads where this node places the module's directories, by directory id. // // It refuses an id the module declares no directory for, a path that is not absolute, and an // owner that is not uid:gid. A directory the assignment does not mention keeps the manifest's // stated path or the node's default layout. func Places(m Manifest, layers []Layer) (map[string]Placement, error) { declared := map[string]bool{} for _, r := range m.Resources { if fmt.Sprint(r["type"]) == "directory" { declared[fmt.Sprint(r["id"])] = true } } out := map[string]Placement{} for _, layer := range layers { raw, ok := layer.Values[PlacesSetting] if !ok { continue } blocks, ok := raw.(map[string]any) if !ok { return nil, fmt.Errorf("%s: %s is a { directory: path | { path, owner } } map, and %q set it to something else", m.Module, PlacesSetting, layer.From) } for id, body := range blocks { if !declared[id] { return nil, fmt.Errorf( "%s places the directory %q, which it does not declare — the setting reaches "+ "nothing. It declares %s", m.Module, id, orNothing(namesOfDirs(directoriesOf(m)))) } p := out[id] switch v := body.(type) { case string: p.Path = strings.TrimSpace(v) case map[string]any: if path, said := v["path"]; said { text, _ := path.(string) p.Path = strings.TrimSpace(text) } if owner, said := v["owner"]; said { text, _ := owner.(string) if !ownerShape.MatchString(strings.TrimSpace(text)) { return nil, fmt.Errorf("%s places %q with owner %v; an owner is uid:gid, numeric", m.Module, id, owner) } p.Owner = strings.TrimSpace(text) } default: return nil, fmt.Errorf("%s places %q with %v; a placement is a path, or { path, owner }", m.Module, id, body) } if p.Path == "" { return nil, fmt.Errorf("%s places %q without a path", m.Module, id) } if !strings.HasPrefix(p.Path, "/") { return nil, fmt.Errorf("%s places %q at %q, which is not an absolute path", m.Module, id, p.Path) } p.Path = filepath.Clean(p.Path) if why := systemPath(p.Path); why != "" { return nil, fmt.Errorf("%s places %q at %s: %s, and the node-engine creates and owns a placed "+ "directory as root, with the owner the setting names — no placement is ever there "+ "(novox/hq issue 339)", m.Module, id, p.Path, why) } out[id] = p } } if len(out) == 0 { return nil, nil } return out, nil } // AccessPlaces reads where this node keeps the operator's data the module accesses, by access id. // // It refuses an id the module declares no access under, and a path that is not absolute. An // access declared by path alone cannot be placed — it has no name to place it by. func AccessPlaces(m Manifest, layers []Layer) (map[string]string, error) { declared := map[string]bool{} for _, a := range m.Accesses { if a.ID != "" { declared[a.ID] = true } } out := map[string]string{} for _, layer := range layers { raw, ok := layer.Values[AccessesSetting] if !ok { continue } blocks, ok := raw.(map[string]any) if !ok { return nil, fmt.Errorf("%s: %s is a { access: path } map, and %q set it to something else", m.Module, AccessesSetting, layer.From) } for id, body := range blocks { if !declared[id] { return nil, fmt.Errorf( "%s places the access %q, which it does not declare — the setting reaches "+ "nothing. It declares %s", m.Module, id, orNothing(namesOfAccesses(m))) } path, _ := body.(string) path = strings.TrimSpace(path) if !strings.HasPrefix(path, "/") { return nil, fmt.Errorf("%s places the access %q at %v, which is not an absolute path", m.Module, id, body) } path = filepath.Clean(path) if why := systemPath(path); why != "" { return nil, fmt.Errorf("%s places the access %q at %s: %s, and an access is mounted into the "+ "module's container — no access is ever there (novox/hq issue 339)", m.Module, id, path, why) } out[id] = path } } if len(out) == 0 { return nil, nil } return out, nil } // placedAccess is one access with the path it resolves to on this node. type placedAccess struct { ID string Path string Mode string } // accessesFor is every access of a module with its path on this node: the assignment's where it // placed one, the definition's where it carries a default, and refused where neither says — an // access that resolves to nowhere would reach the machine as a mount of nothing. func accessesFor(m Manifest, layers []Layer) ([]placedAccess, map[string]string, error) { placed, err := AccessPlaces(m, layers) if err != nil { return nil, nil, err } var out []placedAccess byID := map[string]string{} for _, a := range m.Accesses { path := a.Path if a.ID != "" { if at, said := placed[a.ID]; said { path = at } } if path == "" { return nil, nil, fmt.Errorf( "%s accesses %q, and nothing says where that is on this node — the definition "+ "carries no path (it must not, novox/hq ADR 0112) and the assignment places "+ "none. Set %s: {%q: \"/where/it/is\"}", m.Module, a.ID, AccessesSetting, a.ID) } out = append(out, placedAccess{ID: a.ID, Path: path, Mode: a.At()}) if a.ID != "" { byID[a.ID] = path } } return out, byID, nil } // accessFill resolves every ${access:…} in one string, or refuses a reference naming no access. func accessFill(s string, accesses map[string]string, module string) (string, error) { var missing error out := accessRef.ReplaceAllStringFunc(s, func(ref string) string { id := accessRef.FindStringSubmatch(ref)[1] path, has := accesses[id] if !has { missing = fmt.Errorf( "%s says ${access:%s}, and %s declares no access %q. It declares %s", module, id, module, id, orNothing(namesOfAccessIDs(accesses))) return ref } return path }) return out, missing } // accessInto fills every ${access:…} a resource carries — in its path, its content, its mounts, // its environment and its env-files — with the path this node resolved for it. The same walk as // dirInto, for the same reason: a literal `${access:x}` reaching the machine would be mounted as // a directory called that. func accessInto(resource map[string]any, accesses map[string]string, module string) error { if !mentionsAccess(resource) { return nil } fill := func(s string) (string, error) { return accessFill(s, accesses, module) } var err error if path, ok := resource["path"].(string); ok { if resource["path"], err = fill(path); err != nil { return err } } if content, ok := resource["content"].(string); ok { if resource["content"], err = fill(content); err != nil { return err } } for _, field := range []string{"volumes", "env-file"} { list, ok := resource[field].([]any) if !ok { continue } filled := make([]any, len(list)) for i, v := range list { filled[i] = v if s, ok := v.(string); ok { if filled[i], err = fill(s); err != nil { return err } } } resource[field] = filled } if env, ok := resource["env"].(map[string]any); ok { filled := make(map[string]any, len(env)) for key, v := range env { filled[key] = v if s, ok := v.(string); ok { if filled[key], err = fill(s); err != nil { return err } } } resource["env"] = filled } return nil } func mentionsAccess(resource map[string]any) bool { for _, field := range []string{"path", "content"} { if s, ok := resource[field].(string); ok && accessRef.MatchString(s) { return true } } for _, field := range []string{"volumes", "env-file"} { if list, ok := resource[field].([]any); ok { for _, v := range list { if s, ok := v.(string); ok && accessRef.MatchString(s) { return true } } } } if env, ok := resource["env"].(map[string]any); ok { for _, v := range env { if s, ok := v.(string); ok && accessRef.MatchString(s) { return true } } } return false } // ownerInto gives a placed directory the owner the assignment said, where it said one. The // manifest's owner is what the image expects on any machine; the assignment's is what this // machine's data already is. func ownerInto(resource map[string]any, placed map[string]Placement) { if fmt.Sprint(resource["type"]) != "directory" { return } if p, ok := placed[fmt.Sprint(resource["id"])]; ok && p.Owner != "" { resource["owner"] = p.Owner } } // unknownAccessRefs is every ${access:…} in the definition that names no access the definition // declares by id — refused where the author is, as unknownDirRefs does for directories. func (m Manifest) unknownAccessRefs() []string { declared := map[string]bool{} for _, a := range m.Accesses { if a.ID != "" { declared[a.ID] = true } } seen := map[string]bool{} var problems []string refuse := func(s string, where any) { for _, match := range accessRef.FindAllStringSubmatch(s, -1) { id := match[1] if declared[id] || seen[id] { continue } seen[id] = true problems = append(problems, fmt.Sprintf( "%s says ${access:%s} in %v, and declares no access %q — a reference the mesh "+ "cannot place would reach the machine as a literal path", m.Module, id, where, id)) } } for _, r := range m.Resources { for _, field := range []string{"path", "content"} { if s, ok := r[field].(string); ok { refuse(s, r["id"]) } } for _, field := range []string{"volumes", "env-file"} { if list, ok := r[field].([]any); ok { for _, v := range list { if s, ok := v.(string); ok { refuse(s, r["id"]) } } } } if env, ok := r["env"].(map[string]any); ok { for _, v := range env { if s, ok := v.(string); ok { refuse(s, r["id"]) } } } } sort.Strings(problems) return problems } func directoriesOf(m Manifest) map[string]string { dirs := map[string]string{} for _, r := range m.Resources { if fmt.Sprint(r["type"]) == "directory" { dirs[fmt.Sprint(r["id"])] = "" } } return dirs } func namesOfAccesses(m Manifest) []string { var names []string for _, a := range m.Accesses { if a.ID != "" { names = append(names, fmt.Sprintf("%q", a.ID)) } } sort.Strings(names) return names } func namesOfAccessIDs(accesses map[string]string) []string { var names []string for id := range accesses { names = append(names, fmt.Sprintf("%q", id)) } sort.Strings(names) return names }