Files
mesh-controller/internal/catalogue/placement.go
T
jschoubben 63602264ef Give every module an identity and ask once to keep a shared folder (hq ADR 0325, issue 575)
A module's containers that declare identity run as an identity the controller allocates once, mesh-wide,
from 50000-59999; each access names its users and whether its folder is taken; an access wait is excused
for a folder the mesh may not keep yet, and the operator is asked once per folder and machine, the folder
taken only on the warrant. All of it is sent only to an engine at contract 5. A placement for an access a
newer build brings is judged against that build (issue 575). mesh-host pinned to the ADR 0325 branch head.
2026-10-12 19:05:13 +02:00

569 lines
21 KiB
Go

package catalogue
import (
"errors"
"fmt"
"path/filepath"
"regexp"
"sort"
"strings"
)
// Where a module's data is on THIS machine is the assignment's (novox/hq ADR 0112, issue 153).
//
// A definition names no host path. It declares the directories it owns by id, and the operator's
// shared data it needs by id too (an `access`, ADR 0051). A node has a default layout for the
// former — <root>/<module>/<id> — and nothing at all for the latter, because shared data is
// wherever the operator keeps it. An adopted machine keeps its data where the predecessor put it:
// a 40 TB library on its own pool, a configuration on a second disk. Both halves are said on the
// assignment, validated the way `endpoints` is — an id the module does not declare is refused,
// because a setting that reaches nothing is a mistake — and resolved here, so the host receives
// concrete paths exactly as it always has and learns no field.
//
// {"places": {"config": "/services/sonarr/config",
// "data": {"path": "/mnt/plex/data", "owner": "1000:1000"}},
// "accesses": {"series": "/storage/media/series",
// "downloads": "/storage/downloads"}}
//
// A placed directory is still the mesh's: created, chowned to the owner the assignment says (or
// the manifest's), removed when empty and no longer declared. A placed access is still the
// operator's: mounted, never created, chowned or removed.
// PlacesSetting is the settings key that places a module's declared directories, by id.
const PlacesSetting = "places"
// AccessesSetting is the settings key that says where a module's accesses are on this node, by id.
const AccessesSetting = "accesses"
// Placement is what an assignment says about one of a module's directories.
type Placement struct {
// Path is where the directory is on this machine. Absolute.
Path string
// Owner is "uid:gid" when the assignment overrides the manifest's — the predecessor's data is
// owned by whoever it ran as, and that is one machine's fact.
Owner string
}
var ownerShape = regexp.MustCompile(`^[0-9]+:[0-9]+$`)
// Where no placement and no access may be, from any route, the controller's terminal too (novox/hq issue 339).
//
// A placed directory is created and owned by the node-engine as root, with the owner the setting names, and
// whatever the module writes into it is written as root; an access is mounted into the module's container,
// which may run as root. A place at /etc owned by an account a caller names hands that account the machine,
// and an access at / mounts the machine's root into a container. So the machine's own trees are refused here,
// before anything is kept or composed, and the node-engine refuses them again where it applies.
//
// systemTrees are refused at and below: the machine's system, the kernel's, the boot loader's, root's home,
// what lives only while the machine runs, the spool (cron's tables are there), the container runtimes' data
// (every container's filesystem), /opt, and the node-engine's and the mesh's own state. Any home's .ssh is
// refused as well, wherever the home is. systemRoots are
// refused at, and wherever a path holds one (an ancestor of /var/lib holds it): each is the parent of every
// module's or every person's directories, and owning it is owning all of them.
var (
systemTrees = []string{"/etc", "/usr", "/boot", "/root", "/run", "/var/run", "/var/lock", "/proc", "/sys",
"/dev", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/var/lib/mesh", "/var/lib/mesh-host", "/var/spool",
"/var/lib/docker", "/var/lib/containers", "/var/lib/containerd", "/opt"}
systemRoots = []string{"/", "/var", "/var/lib", "/var/cache", "/var/log", "/var/tmp", "/home",
"/mnt", "/media", "/srv", "/tmp", "/storage", "/data", "/services"}
)
// systemPath says why a clean absolute path is the machine's own and never a placement's or an access's, or "".
func systemPath(path string) string {
// Any home's keys, wherever the home is: a .ssh directory is its account's, and the keys and the list of who
// may log in as it are in there.
for _, part := range strings.Split(path, "/") {
if part == ".ssh" {
return path + " is an account's .ssh, which holds its keys and who may log in as it"
}
}
for _, tree := range systemTrees {
if path == tree || strings.HasPrefix(path, tree+"/") {
return path + " is in " + tree + ", the machine's own or the mesh's state"
}
if strings.HasPrefix(tree, path+"/") || path == "/" {
return path + " holds " + tree + ", the machine's own or the mesh's state"
}
}
for _, root := range systemRoots {
if path == root {
return path + " is the parent of every module's or every person's directories"
}
}
return ""
}
// Where no directory or file a module's definition resolves to may be (novox/hq issue 496).
//
// mesh-catalog #205 gave docker's `state` directory `"place": "."`, which resolves to <root>/docker: with the
// default root, /var/lib/docker, every container's filesystem. systemPath judged only the `places` and `accesses`
// settings, so the default layout and a definition's own paths reached the merge gate unjudged; it composed every
// machine and passed, and only the node-engine refused the directory, at apply, failing the walk.
//
// **Judged where a definition is judged, never where a machine is composed.** resolvedPathProblems runs in
// ParseManifest, which every route a definition takes into the mesh passes: `module check`, registration of a
// build (the builder's and the registry verbs'), and the merge gate's reading of the changed repository. A
// refusal there stops one definition before it reaches any machine. Composition reads registered manifests
// without ParseManifest, and judges nothing of this: refusing there would fail the whole machine's declaration and
// freeze every module on it for one module's path, where the node-engine fails only that resource.
//
// **The node-engine's rules, no more** (mesh-host's internal/apply/placement_guard.go, with files judged as
// directories are after novox/hq issue 495, rule 6). A path is refused when it is one of protectedRoots or holds
// one, when it is at or below a tree in forbiddenBelow, or at or below one of engineTrees and its module is not
// the node-engine's. What the engine judges with what only the machine knows stays the engine's: where the
// runtimes really keep their data, links, the accounts' homes, a directory's owner below /etc. The lists are the
// engine's own words, and a test (engine_guard_test.go) holds them equal to mesh-host's beside this repository.
// systemPath stays the stricter rule for a setting: a setting is an operator's word about one machine, and the
// trees it lists (/etc, /usr, /run, the mesh's own) are where the mesh's own modules write by design.
var (
protectedRoots = []string{"/", "/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib32", "/lib64",
"/media", "/mnt", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/usr/bin",
"/usr/lib", "/usr/lib64", "/usr/local", "/usr/local/bin", "/usr/local/lib", "/usr/local/sbin", "/usr/sbin",
"/usr/share", "/var", "/var/cache", "/var/lib", "/var/lib/mesh", "/var/log", "/var/run", "/var/tmp",
"/var/spool"}
forbiddenBelow = []string{"/proc", "/sys", "/dev", "/boot", "/root", "/var/spool", "/opt", "/var/lib/docker",
"/var/lib/containers", "/var/lib/containerd"}
engineTrees = []string{"/var/lib/mesh-host", "/usr/lib/nox-mesh-host"}
)
// engineModule is the node-engine's own module, the one that places in engineTrees.
const engineModule = "mesh-host"
// enginePath says why the node-engine refuses a directory or file at path for module, or "".
func enginePath(path, module string) string {
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
return ""
}
atOrBelow := func(tree string) bool { return path == tree || strings.HasPrefix(path, tree+"/") }
for _, root := range protectedRoots {
if path == root || path == "/" || strings.HasPrefix(root, path+"/") {
return root + " is one of the machine's own directories, and owning it is owning everything in it"
}
}
for _, tree := range forbiddenBelow {
if atOrBelow(tree) {
return "nothing is placed in " + tree
}
}
if module != engineModule {
for _, tree := range engineTrees {
if atOrBelow(tree) {
return tree + " is the node-engine's own, placed in by its own module alone"
}
}
}
return ""
}
// resolvedPathProblems is every directory and file of the definition whose path, resolved as a node with the
// default root resolves it, the node-engine would refuse (novox/hq issue 496). A path still holding a placeholder
// only a machine fills (a setting, an access the definition gives no default) is the engine's to judge.
func (m Manifest) resolvedPathProblems() []string {
dirs := dirsFor(m, Rendering{})
accesses := map[string]string{}
for _, a := range m.Accesses {
if a.ID != "" && a.Path != "" {
accesses[a.ID] = a.Path
}
}
var problems []string
for _, r := range m.Resources {
kind := fmt.Sprint(r["type"])
if kind != "directory" && kind != "file" {
continue
}
id := fmt.Sprint(r["id"])
path, _ := r["path"].(string)
if kind == "directory" && path == "" {
path = dirs[id]
}
path, _ = dirFill(path, dirs, m.Module)
path, _ = accessFill(path, accesses, m.Module)
if path == "" || strings.Contains(path, "${") {
continue
}
if why := enginePath(path, m.Module); why != "" {
problems = append(problems, fmt.Sprintf("%s's %s %q resolves to %s, which the node-engine refuses: %s. "+
"A module's directories and files are judged when its definition is, so the merge gate refuses it "+
"before any machine does (novox/hq issue 496)", m.Module, kind, id, filepath.Clean(path), why))
}
}
return problems
}
// accessRef is how a module names one of its accesses: ${access:<id>}.
var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`)
// Places reads where this node places the module's directories, by directory id.
//
// It refuses an id the module declares no directory for, a path that is not absolute, and an
// owner that is not uid:gid. A directory the assignment does not mention keeps the manifest's
// stated path or the node's default layout.
func Places(m Manifest, layers []Layer) (map[string]Placement, error) {
declared := map[string]bool{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "directory" {
declared[fmt.Sprint(r["id"])] = true
}
}
out := map[string]Placement{}
for _, layer := range layers {
raw, ok := layer.Values[PlacesSetting]
if !ok {
continue
}
blocks, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: %s is a { directory: path | { path, owner } } map, and %q set it to something else",
m.Module, PlacesSetting, layer.From)
}
for id, body := range blocks {
if !declared[id] {
return nil, fmt.Errorf(
"%s places the directory %q, which it does not declare — the setting reaches "+
"nothing. It declares %s", m.Module, id, orNothing(namesOfDirs(directoriesOf(m))))
}
p := out[id]
switch v := body.(type) {
case string:
p.Path = strings.TrimSpace(v)
case map[string]any:
if path, said := v["path"]; said {
text, _ := path.(string)
p.Path = strings.TrimSpace(text)
}
if owner, said := v["owner"]; said {
text, _ := owner.(string)
if !ownerShape.MatchString(strings.TrimSpace(text)) {
return nil, fmt.Errorf("%s places %q with owner %v; an owner is uid:gid, numeric",
m.Module, id, owner)
}
p.Owner = strings.TrimSpace(text)
}
default:
return nil, fmt.Errorf("%s places %q with %v; a placement is a path, or { path, owner }",
m.Module, id, body)
}
if p.Path == "" {
return nil, fmt.Errorf("%s places %q without a path", m.Module, id)
}
if !strings.HasPrefix(p.Path, "/") {
return nil, fmt.Errorf("%s places %q at %q, which is not an absolute path", m.Module, id, p.Path)
}
p.Path = filepath.Clean(p.Path)
if why := systemPath(p.Path); why != "" {
return nil, fmt.Errorf("%s places %q at %s: %s, and the node-engine creates and owns a placed "+
"directory as root, with the owner the setting names — no placement is ever there "+
"(novox/hq issue 339)", m.Module, id, p.Path, why)
}
out[id] = p
}
}
if len(out) == 0 {
return nil, nil
}
return out, nil
}
// AccessPlaces reads where this node keeps the operator's data the module accesses, by access id.
//
// It refuses an id the module declares no access under, and a path that is not absolute. An
// access declared by path alone cannot be placed — it has no name to place it by.
func AccessPlaces(m Manifest, layers []Layer) (map[string]string, error) {
return accessPlaces(m, layers, true)
}
// UndeclaredAccessError is a layer placing an access the definition does not declare (novox/hq issue 575): refused
// where a setting is stored, unless a newer build of the module declares it; passed by where a machine is composed.
type UndeclaredAccessError struct {
Module, ID string
said string
}
func (e *UndeclaredAccessError) Error() string { return e.said }
// UndeclaredAccessPlaces refuses a layer that places an access the definition does not declare.
func UndeclaredAccessPlaces(m Manifest, layers []Layer) error {
_, err := accessPlaces(m, layers, true)
var undeclared *UndeclaredAccessError
if errors.As(err, &undeclared) {
return err
}
return nil
}
// accessPlaces is AccessPlaces; where strict is false an access the definition does not declare is passed by
// (novox/hq issue 575), as a placement stored ahead of the build that declares it.
func accessPlaces(m Manifest, layers []Layer, strict bool) (map[string]string, error) {
declared := map[string]bool{}
for _, a := range m.Accesses {
if a.ID != "" {
declared[a.ID] = true
}
}
out := map[string]string{}
for _, layer := range layers {
raw, ok := layer.Values[AccessesSetting]
if !ok {
continue
}
blocks, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s: %s is a { access: path } map, and %q set it to something else",
m.Module, AccessesSetting, layer.From)
}
for id, body := range blocks {
if !declared[id] {
if !strict {
continue
}
return nil, &UndeclaredAccessError{Module: m.Module, ID: id, said: fmt.Sprintf(
"%s places the access %q, which it does not declare — the setting reaches "+
"nothing. It declares %s", m.Module, id, orNothing(namesOfAccesses(m)))}
}
path, _ := body.(string)
path = strings.TrimSpace(path)
if !strings.HasPrefix(path, "/") {
return nil, fmt.Errorf("%s places the access %q at %v, which is not an absolute path",
m.Module, id, body)
}
path = filepath.Clean(path)
if why := systemPath(path); why != "" {
return nil, fmt.Errorf("%s places the access %q at %s: %s, and an access is mounted into the "+
"module's container — no access is ever there (novox/hq issue 339)", m.Module, id, path, why)
}
out[id] = path
}
}
if len(out) == 0 {
return nil, nil
}
return out, nil
}
// placedAccess is one access with the path it resolves to on this node.
type placedAccess struct {
ID string
Path string
Mode string
}
// accessesFor is every access of a module with its path on this node: the assignment's where it
// placed one, the definition's where it carries a default, and refused where neither says — an
// access that resolves to nowhere would reach the machine as a mount of nothing.
func accessesFor(m Manifest, layers []Layer) ([]placedAccess, map[string]string, error) {
placed, err := accessPlaces(m, layers, false)
if err != nil {
return nil, nil, err
}
var out []placedAccess
byID := map[string]string{}
for _, a := range m.Accesses {
path := a.Path
if a.ID != "" {
if at, said := placed[a.ID]; said {
path = at
}
}
if path == "" {
return nil, nil, fmt.Errorf(
"%s accesses %q, and nothing says where that is on this node — the definition "+
"carries no path (it must not, novox/hq ADR 0112) and the assignment places "+
"none. Set %s: {%q: \"/where/it/is\"}",
m.Module, a.ID, AccessesSetting, a.ID)
}
out = append(out, placedAccess{ID: a.ID, Path: path, Mode: a.At()})
if a.ID != "" {
byID[a.ID] = path
}
}
return out, byID, nil
}
// accessFill resolves every ${access:…} in one string, or refuses a reference naming no access.
func accessFill(s string, accesses map[string]string, module string) (string, error) {
var missing error
out := accessRef.ReplaceAllStringFunc(s, func(ref string) string {
id := accessRef.FindStringSubmatch(ref)[1]
path, has := accesses[id]
if !has {
missing = fmt.Errorf(
"%s says ${access:%s}, and %s declares no access %q. It declares %s",
module, id, module, id, orNothing(namesOfAccessIDs(accesses)))
return ref
}
return path
})
return out, missing
}
// accessInto fills every ${access:…} a resource carries — in its path, its content, its mounts,
// its environment and its env-files — with the path this node resolved for it. The same walk as
// dirInto, for the same reason: a literal `${access:x}` reaching the machine would be mounted as
// a directory called that.
func accessInto(resource map[string]any, accesses map[string]string, module string) error {
if !mentionsAccess(resource) {
return nil
}
fill := func(s string) (string, error) { return accessFill(s, accesses, module) }
var err error
if path, ok := resource["path"].(string); ok {
if resource["path"], err = fill(path); err != nil {
return err
}
}
if content, ok := resource["content"].(string); ok {
if resource["content"], err = fill(content); err != nil {
return err
}
}
for _, field := range []string{"volumes", "env-file"} {
list, ok := resource[field].([]any)
if !ok {
continue
}
filled := make([]any, len(list))
for i, v := range list {
filled[i] = v
if s, ok := v.(string); ok {
if filled[i], err = fill(s); err != nil {
return err
}
}
}
resource[field] = filled
}
if env, ok := resource["env"].(map[string]any); ok {
filled := make(map[string]any, len(env))
for key, v := range env {
filled[key] = v
if s, ok := v.(string); ok {
if filled[key], err = fill(s); err != nil {
return err
}
}
}
resource["env"] = filled
}
return nil
}
func mentionsAccess(resource map[string]any) bool {
for _, field := range []string{"path", "content"} {
if s, ok := resource[field].(string); ok && accessRef.MatchString(s) {
return true
}
}
for _, field := range []string{"volumes", "env-file"} {
if list, ok := resource[field].([]any); ok {
for _, v := range list {
if s, ok := v.(string); ok && accessRef.MatchString(s) {
return true
}
}
}
}
if env, ok := resource["env"].(map[string]any); ok {
for _, v := range env {
if s, ok := v.(string); ok && accessRef.MatchString(s) {
return true
}
}
}
return false
}
// ownerInto gives a placed directory the owner the assignment said, where it said one. The
// manifest's owner is what the image expects on any machine; the assignment's is what this
// machine's data already is.
func ownerInto(resource map[string]any, placed map[string]Placement) {
if fmt.Sprint(resource["type"]) != "directory" {
return
}
if p, ok := placed[fmt.Sprint(resource["id"])]; ok && p.Owner != "" {
resource["owner"] = p.Owner
}
}
// unknownAccessRefs is every ${access:…} in the definition that names no access the definition
// declares by id — refused where the author is, as unknownDirRefs does for directories.
func (m Manifest) unknownAccessRefs() []string {
declared := map[string]bool{}
for _, a := range m.Accesses {
if a.ID != "" {
declared[a.ID] = true
}
}
seen := map[string]bool{}
var problems []string
refuse := func(s string, where any) {
for _, match := range accessRef.FindAllStringSubmatch(s, -1) {
id := match[1]
if declared[id] || seen[id] {
continue
}
seen[id] = true
problems = append(problems, fmt.Sprintf(
"%s says ${access:%s} in %v, and declares no access %q — a reference the mesh "+
"cannot place would reach the machine as a literal path",
m.Module, id, where, id))
}
}
for _, r := range m.Resources {
for _, field := range []string{"path", "content"} {
if s, ok := r[field].(string); ok {
refuse(s, r["id"])
}
}
for _, field := range []string{"volumes", "env-file"} {
if list, ok := r[field].([]any); ok {
for _, v := range list {
if s, ok := v.(string); ok {
refuse(s, r["id"])
}
}
}
}
if env, ok := r["env"].(map[string]any); ok {
for _, v := range env {
if s, ok := v.(string); ok {
refuse(s, r["id"])
}
}
}
}
sort.Strings(problems)
return problems
}
func directoriesOf(m Manifest) map[string]string {
dirs := map[string]string{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "directory" {
dirs[fmt.Sprint(r["id"])] = ""
}
}
return dirs
}
func namesOfAccesses(m Manifest) []string {
var names []string
for _, a := range m.Accesses {
if a.ID != "" {
names = append(names, fmt.Sprintf("%q", a.ID))
}
}
sort.Strings(names)
return names
}
func namesOfAccessIDs(accesses map[string]string) []string {
var names []string
for id := range accesses {
names = append(names, fmt.Sprintf("%q", id))
}
sort.Strings(names)
return names
}