// Package declaration is what the host is told a machine should be. // // Data, never instructions. The vocabulary is finite, versioned, and anything outside it // refuses the whole declaration rather than being skipped — a host that applied most of what // it was sent and reported success is a node that looks configured and is not // (novox/hq ADR 0005). package declaration import ( "bytes" "encoding/json" "fmt" "io" "reflect" "regexp" "slices" "sort" "strings" ) // Version is the vocabulary this host speaks. A declaration naming any other version is // refused: an older host handed a newer vocabulary must not quietly do half of it. const Version = 1 // Type names a kind of resource. Every addition widens what a compromised control plane can // express, so the list is a security artefact and grows deliberately. type Type string const ( TypeDirectory Type = "directory" TypeFile Type = "file" TypeService Type = "service" TypePackage Type = "package" TypeContainer Type = "container" TypeAction Type = "action" // TypeUser is a login on the machine. Added because most of what a person actually installs // is not a service: a shell, a terminal, a chat client, a desktop. All of those are a package // plus configuration **in somebody's home**, and a mesh with no notion of a user can only // manage /etc. TypeUser Type = "user" // TypeArchive is a set of files fetched by digest and unpacked. A desktop theme is hundreds // of files; inlining them would make every declaration enormous and rewrite the lot whenever // one changed. TypeArchive Type = "archive" // TypeNetwork is a named network on this machine, for a module whose containers must reach // each other by name. Created if absent, removed when no longer declared — which is the whole // reason it is a shape rather than an action, because an action leaves nothing the host can // undo and the network would outlive the module (novox/hq ADR 0029). TypeNetwork Type = "network" // TypeAccess is a pre-existing, operator-owned path a module is granted use of but does not // own (novox/hq ADR 0051). The opposite of a directory on every axis the host acts on: the // host creates, chowns and reconciles a directory, and removes it when it is empty; it does // none of that to an access. It confirms the path is present — refusing clearly if the // operator has not provided it, rather than creating it as a bind mount source would // (04-ISSUES/026) — and leaves everything about it alone. Several modules declaring one // access is ordinary, because none of them owns it. TypeAccess Type = "access" // TypeProcess is the module's own code, run on the machine, in one of three modes. // // **The intent, not the mechanism.** Until this, an author decided the hosting before they // could declare anything: code of their own meant a `container` built from an image, a script // meant a `service` and a unit somebody else had to install. Same intent — run this — with // the choice baked into which kind was picked. // // **And three modes rather than three kinds**, exactly as a container has. A first draft of // this added a `daemon` for the long-running case alone, which would have meant a new kind for // each of the others — a scheduled task, a run-once migration, a health check. They are one // thing run at different cadences, and that is a field, not a vocabulary entry. Every addition // to this vocabulary widens what a compromised control plane can express. // // What a module declares is a bundle and a command. The mesh unpacks the bundle where it keeps // such things and runs it — as a unit that stays up, as a step that must finish, or on a // cadence. Tools, hooks and event consumers are not separate modes: they are loaded by a tool // host, which is itself a process that stays up. TypeProcess Type = "process" // TypeOpening is a port the mesh needs reachable on an adopted node, converged through the // firewall found there in that firewall's own terms (novox/hq ADR 0100). A state, not a // command: the host adds the rule it marks as the mesh's when it is missing, and removes only // what it marked — which is what lets it travel over the link. TypeOpening Type = "opening" ) // Resource is one thing that should be true of the machine. // // A struct per kind rather than one struct carrying every field, because the decoder is then // what rejects a field the kind does not have: a `file` carrying an `image` is refused because // File has no such field, not because a list somewhere remembered to say so. The one-struct // form needs every kind revisited whenever a field is added, and the kind nobody revisits // silently accepts a field the host will never read. type Resource interface { // Identity is the name the control plane keeps stable across declarations. Not a position // and not a hash of the content: it is what lets the store say *this is the same resource // I applied last time*, which is what makes removal possible at all. Identity() string // Kind is the resource's type, for the store and for reporting. Kind() Type // Target is what the resource acts on, for a person reading a report. Target() string validate(where string, allowActions bool) []string } // The `Type` field on each kind below exists only to absorb the JSON `"type"` key, which the // strict decoder would otherwise refuse. `Kind()` returns the constant and is what anything // else should read. // Directory is a directory that should exist, with a mode. type Directory struct { ID string `json:"id"` Type Type `json:"type"` Path string `json:"path"` Mode string `json:"mode,omitempty"` // Owner is the user this belongs to, by name. Absent means root, which is what everything // managed was until users existed. Owner string `json:"owner,omitempty"` } func (d *Directory) Identity() string { return d.ID } func (d *Directory) Kind() Type { return TypeDirectory } func (d *Directory) Target() string { return d.Path } func (d *Directory) validate(where string, _ bool) []string { var problems []string if d.Path == "" { problems = append(problems, where+": a directory needs a path") } return append(problems, checkMode(where, d.Mode)...) } // File is a file with literal content. The host renders nothing. type File struct { ID string `json:"id"` Type Type `json:"type"` Path string `json:"path"` Content string `json:"content"` Mode string `json:"mode,omitempty"` // CreateOnce says the content is a seed: written when the file is absent, and left alone — // content, mode and owner — whenever it is present. // // **Two intentions had one vocabulary** (novox/hq issue 035, ADR 0087). "This file has this // content, for ever" is what an ordinary file says, and the host holds the machine to it. A // module that needs a file to exist before a program first starts — an access list the // program then persists into, a bootstrap configuration it rewrites — needs the other thing, // and with only the first available, every reconcile restored the seed behind the running // program and erased what had grown in it, reporting success. What grows in a seeded file // is somebody else's work the mesh asked for; the mesh removes nothing it did not create // (ADR 0030), and it does not overwrite that either. CreateOnce bool `json:"create-once,omitempty"` // Into says the file is shared with software the mesh did not install, and the content is // the mesh's part of it: written into what is there, never over it (novox/hq ADR 0102). Only // "json" is spoken — the content is a JSON object whose keys the host sets in the file's // object, keeping every other key as it found it and recording what each of its keys held // before, so undeclaring the file gives those back. Into string `json:"into,omitempty"` // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver // without being able to read. // // The one thing here the host cannot simply write. Everything else in a declaration is // visible to whatever carried it — the broker relays the message, and the message is signed // so it cannot be forged, but signing does not make it unreadable. A password travelling in // `content` would be a password the broker sees, which is the transitive trust the design // refuses everywhere else (novox/hq ADR 0004). // // Exclusive with Content: a file is one or the other, so that "was this secret" is answerable // by looking rather than by knowing which field won. Sealed string `json:"sealed,omitempty"` // Secrets are sealed values put into Content where it says `${secret:name}`. // // **The one place a secret and a configuration meet, and it happens on the machine.** A // program that wants its token inside a JSON document cannot be given a file that is entirely // a token, and the mesh cannot compose the document itself — it discarded the value // (novox/hq ADR 0024). So the module supplies the document with a hole in it, the mesh // delivers the value sealed, and the host is the only thing that ever sees both. // // **Substitution is textual and the host learns no formats.** That is deliberate: a mechanism // that understood JSON would be asked to understand YAML next, and then INI, which is how the // arrangement this replaces became something nobody could hold in their head. The module knows // its own format, because it wrote the rest of the file. // // The sharp edge, stated rather than discovered: a value containing a quote or a backslash // will not be escaped for whatever syntax surrounds it. Secrets map[string]string `json:"secrets,omitempty"` // Bytes is content that is not text, base64-encoded — a wallpaper, a font, an icon. // // A third way of saying what is in a file, and the three are exclusive. It would have been // tempting to let Content carry base64 and add a flag, and then "what is in this file" would // depend on a field somewhere else. Bytes string `json:"bytes,omitempty"` // Owner is the user this belongs to, by name. Absent means root. Owner string `json:"owner,omitempty"` } // Secret reports whether this file arrived sealed, which is what decides both that it must be // opened before writing and that its contents must never appear in a report. func (f *File) Secret() bool { return f.Sealed != "" } func (f *File) Identity() string { return f.ID } func (f *File) Kind() Type { return TypeFile } func (f *File) Target() string { return f.Path } // placeholder is what Content says where a sealed value belongs: ${secret:name}. var placeholder = regexp.MustCompile(`\$\{secret:([a-z0-9][a-z0-9-]*)\}`) // SecretsUsed are the names Content asks for, in the order they first appear. func (f *File) SecretsUsed() []string { var used []string seen := map[string]bool{} for _, m := range placeholder.FindAllStringSubmatch(f.Content, -1) { if !seen[m[1]] { seen[m[1]] = true used = append(used, m[1]) } } return used } func (f *File) validate(where string, _ bool) []string { var problems []string if f.Path == "" { problems = append(problems, where+": a file needs a path") } switch f.Into { case "": case IntoJSON: var object map[string]json.RawMessage if err := json.Unmarshal([]byte(f.Content), &object); err != nil || object == nil { problems = append(problems, where+ ": a file written into JSON carries a JSON object of the keys it sets") } if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { problems = append(problems, where+ ": a file written into says only its keys, in content — not sealed, bytes, "+ "secrets or create-once") } default: problems = append(problems, fmt.Sprintf( "%s: into %q; a file is written into \"json\", or omits it to be written whole", where, f.Into)) } var said []string for name, value := range map[string]string{ "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, } { if value != "" { said = append(said, name) } } if len(said) > 1 { sort.Strings(said) problems = append(problems, where+ ": a file says what is in it exactly once, and this says it as "+ strings.Join(said, " and ")+ " — otherwise nobody can tell by looking which one landed on the machine") } // A file whose content names a secret must be given exactly the secrets it names. // // **Both directions, and both are refusals rather than warnings.** A placeholder with nothing // to fill it would write `${secret:x}` into a configuration file, which the program reads as // a value and fails on somewhere unrelated. A secret nobody uses means whoever wrote this // believes a credential is in a file where it is not. if len(f.Secrets) > 0 && f.Content == "" { problems = append(problems, where+ ": secrets were given and there is no content to put them in") } used := f.SecretsUsed() for _, name := range used { if f.Secrets[name] == "" { problems = append(problems, fmt.Sprintf( "%s: the content asks for the secret %q and none was given", where, name)) } } for name := range f.Secrets { if !slices.Contains(used, name) { problems = append(problems, fmt.Sprintf( "%s: the secret %q was given and the content never asks for it", where, name)) } } return append(problems, checkMode(where, f.Mode)...) } // User is a login on the machine. // // The thing that makes a shell, a chat client or a desktop expressible at all: each is a package // plus configuration in somebody's home, and until this the mesh could only own /etc. // // It also makes "zsh is my login shell" **declared state** rather than an action. `chsh` is a // command, the link may not carry one (novox/hq ADR 0005), and a shell that could only be set by // hand would be a shell the mesh cannot manage — which is most of the reason to manage a machine // at all. type User struct { ID string `json:"id"` Type Type `json:"type"` Name string `json:"name"` // Shell this user logs in with. Absent means the host asserts nothing and leaves whatever is // there — the same rule Service.Boot follows, for the same reason: a field that always // asserts cannot express "I do not care". Shell string `json:"shell,omitempty"` // Groups this user must be in. Additive: the host puts the user in these and does not remove // it from others, because a machine's own groups are not the mesh's to know about. Groups []string `json:"groups,omitempty"` // Home directory. Absent means the system's default for a new user, and is not changed for // one that exists — moving somebody's home is not something a declaration should do quietly. Home string `json:"home,omitempty"` } // Network is a named network on this machine. // // **A name and nothing else.** Not a driver, a subnet or a gateway: each of those is something a // module would have to know about the machine it lands on, and a module naming a subnet is a // module that collides with whatever else chose the same one. The runtime picks; the mesh names // (novox/hq ADR 0029). type Network struct { ID string `json:"id"` Type Type `json:"type"` Name string `json:"name"` } func (n *Network) Identity() string { return n.ID } func (n *Network) Kind() Type { return TypeNetwork } func (n *Network) Target() string { return n.Name } func (n *Network) validate(where string, _ bool) []string { var problems []string if n.Name == "" { problems = append(problems, where+": a network needs a name") } // The runtimes accept more than this, and the mesh does not: a name with a slash or a colon // in it reads as a reference to something else entirely wherever it is later printed. for _, r := range n.Name { if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '-' && r != '_' && r != '.' { problems = append(problems, where+ ": a network name is letters, digits, dashes, underscores and dots, and "+ n.Name+" is not") break } } return problems } // Modes an access may be granted at. Plain words, not the octal a directory's mode is: an access // is not a thing the host chmods, it is a statement of how this module reaches what the operator // owns. const ( AccessRead = "read" AccessReadWrite = "read-write" ) // Access is a pre-existing, operator-owned path this module is granted use of but does not own. // // **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own — // it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A // media library, a download spool is the operator's: it existed before the mesh, several modules // read and write it at once, and the mesh must not create, chown, reconcile or remove it. The // host confirms it is there and mounts it; nothing else. type Access struct { ID string `json:"id"` Type Type `json:"type"` Path string `json:"path"` // Mode is how this module reaches the path: read or read-write. Absent narrows to read. Mode string `json:"mode,omitempty"` } func (a *Access) Identity() string { return a.ID } func (a *Access) Kind() Type { return TypeAccess } func (a *Access) Target() string { return a.Path } func (a *Access) validate(where string, _ bool) []string { var problems []string if !strings.HasPrefix(a.Path, "/") { problems = append(problems, where+": an access needs an absolute path, and "+ a.Path+" is not one") } switch a.Mode { case "", AccessRead, AccessReadWrite: default: problems = append(problems, fmt.Sprintf( "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) } return problems } func (u *User) Identity() string { return u.ID } func (u *User) Kind() Type { return TypeUser } func (u *User) Target() string { return u.Name } func (u *User) validate(where string, _ bool) []string { var problems []string if u.Name == "" { problems = append(problems, where+": a user needs a name") } if u.Shell != "" && !strings.HasPrefix(u.Shell, "/") { problems = append(problems, where+ ": a login shell is an absolute path, and "+u.Shell+" is not one") } if u.Home != "" && !strings.HasPrefix(u.Home, "/") { problems = append(problems, where+": a home directory is an absolute path") } return problems } // Archive is a set of files, fetched by digest and unpacked. // // For the case inlining cannot serve: a theme, an icon set, a tree of configuration. Hundreds of // files inlined would make every declaration enormous and rewrite all of it when one changed. // // **Pinned by digest, and the digest is checked before anything is unpacked.** The same discipline // the bootstrap uses for images, and for the same reason — this is fetched over a network the // mesh does not control, and a reference that can be made to point elsewhere is not a reference. type Archive struct { ID string `json:"id"` Type Type `json:"type"` // Source is where to fetch it from. Source string `json:"source"` // Digest is sha256 of the archive, as "sha256:". Digest string `json:"digest"` // Path is the directory it is unpacked into. Path string `json:"path"` // Owner is the user the unpacked files belong to. Absent means root. Owner string `json:"owner,omitempty"` } func (a *Archive) Identity() string { return a.ID } func (a *Archive) Kind() Type { return TypeArchive } func (a *Archive) Target() string { return a.Path } func (a *Archive) validate(where string, _ bool) []string { var problems []string if a.Source == "" { problems = append(problems, where+": an archive needs somewhere to fetch it from") } if a.Path == "" { problems = append(problems, where+": an archive needs somewhere to unpack into") } if !strings.HasPrefix(a.Digest, "sha256:") || len(a.Digest) != len("sha256:")+64 { // Refused rather than fetched and trusted. Everything else pinned in this vocabulary is // pinned by digest, and an archive that was not would be the one way in. problems = append(problems, where+ ": an archive is pinned by digest, as sha256:<64 hex characters>") } return problems } // Process is the module's own code, run on the machine, in one of three modes. // // The difference from Service is who owns the unit: a Service puts an EXISTING unit into a state // and deliberately does not install one, which is right for software that ships its own. This is // the mesh's own code — a bundle it built — so there is no unit until the mesh writes it, and // nothing else will. // // The difference from Container is the hosting, and a module should not have to choose: what this // says is what to run, and the machine's own supervisor is how. Code that genuinely needs a // container's isolation declares a container and says so. // // **Three modes, matching a container's**, because they are the same thing at different cadences: // stays up, runs once, runs on a schedule. A module's scheduled task, its run-once migration, its // health check and its tool host are all this — and each being its own resource kind would be four // entries in a vocabulary where every entry widens what a compromised control plane can express. type Process struct { ID string `json:"id"` Type Type `json:"type"` // Name is what the unit is called, and what an operator will see in the process table. Name string `json:"name"` // Source is where to fetch the bundle from, and Digest is what it must hash to. The same // discipline as an archive, for the same reason: this crosses a network the mesh does not // control. Source string `json:"source"` Digest string `json:"digest"` // Run is the command, relative to the unpacked bundle. The first element is the program. // // **Named by the module, never inferred.** Guessing an entrypoint from which files exist makes // a daemon change what it runs when somebody adds a file. Run []string `json:"run"` // Env and EnvFile are what it runs with. A file rather than inline values is how a credential // reaches a daemon without passing through the declaration. Env map[string]string `json:"env,omitempty"` EnvFile []string `json:"env-file,omitempty"` // User is who it runs as. Absent means root, which is what the mesh's own modules need for // the things they do to a machine. User string `json:"user,omitempty"` // RestartOn names resources whose change means this must be restarted — the same rule a // service follows, and for the same reason: a running process does not re-read its // configuration, so replacing a file and finding the process already up leaves the machine // behaving the way it did before while every check passes. RestartOn []string `json:"restart-on,omitempty"` // RunOnce marks code the host runs to completion rather than leaves running: a migration, a // seed, a first-boot step. What follows it is gated on it finishing, because a step that did // not make the machine ready must not be followed by the thing that needed it. RunOnce bool `json:"run-once,omitempty"` // Schedule runs it on a cadence — a five-field cron expression (novox/hq ADR 0053). The // recurring twin of RunOnce: the same code, run again rather than left running. // // Exclusive with RunOnce and with RestartOn, for the same reason a container's is: something // that runs once does not run on a schedule, and something that is not running cannot be // restarted when a file changes. Schedule string `json:"schedule,omitempty"` } func (d *Process) Identity() string { return d.ID } func (d *Process) Kind() Type { return TypeProcess } func (d *Process) Target() string { return d.Name } func (d *Process) validate(where string, _ bool) []string { var problems []string if d.Name == "" { problems = append(problems, where+": a process needs a name, which is what its unit is called") } if strings.ContainsAny(d.Name, "/ \t") { // It becomes a unit name and a file on disk. A name with a separator in it would write // somewhere nobody meant. problems = append(problems, where+": a process name becomes a unit name, so it cannot "+ "contain a path separator or a space") } if d.Source == "" { problems = append(problems, where+": a process needs somewhere to fetch its bundle from") } if !strings.HasPrefix(d.Digest, "sha256:") || len(d.Digest) != len("sha256:")+64 { // The same rule an archive follows, and for the same reason: this crosses a network the // mesh does not control, and a reference that can be made to point elsewhere is not one. problems = append(problems, where+ ": a process bundle is pinned by digest, as sha256:<64 hex characters>") } if len(d.Run) == 0 { problems = append(problems, where+": a process needs to say what to run") } for _, part := range d.Run { if part == "" { problems = append(problems, where+": a process command has an empty element") break } } // **A newline cannot be represented in a unit's environment, so it is refused rather than // mangled.** Everything else a unit file reinterprets — a percent specifier, whitespace // splitting assignments, a quote ending one early — can be escaped. A newline cannot: it ends // the line, and what follows is read as a unit DIRECTIVE. A value carrying one could write // ExecStart= and have the machine run something nobody declared. // // Refused here, near whoever wrote it, rather than at the far end of a declaration. for key, value := range d.Env { if strings.ContainsAny(value, "\n\r") { problems = append(problems, fmt.Sprintf( "%s: the value of %s contains a line break, which cannot be written into a unit's "+ "environment — what followed it would be read as a unit directive", where, key)) } if key == "" { problems = append(problems, where+": an environment value with no name") } } if d.Schedule != "" { if d.RunOnce { problems = append(problems, where+ ": a process runs once or on a schedule, not both") } if len(d.RestartOn) > 0 { problems = append(problems, where+ ": a scheduled process is not running between its fires, so there is nothing to "+ "restart when something it reads changes") } if _, err := ParseCron(d.Schedule); err != nil { problems = append(problems, where+": "+err.Error()) } } return problems } // Service is a unit the host puts into a state. It does not install the unit. // // Two states, and they are orthogonal rather than one scale. A unit can be enabled and stopped // (it will come back at boot), or disabled and running (started by hand, gone after a reboot). // Folding them into one field would make the second expressible only by accident. type Service struct { ID string `json:"id"` Type Type `json:"type"` Unit string `json:"unit"` State string `json:"state"` // Boot is "enabled" or "disabled" — whether the unit starts at boot. Optional: absent means // the host asserts nothing about it and leaves whatever is there. // // Without this the host could start a unit and not make it survive a reboot, which is a // declaration that reports success and stops being true at the next power cut. Boot string `json:"boot,omitempty"` // RestartOn names resources whose change means this service must be restarted. // // Because a running service does not re-read its configuration. Replace the file, find the // service already running, do nothing, and the machine keeps behaving the way it did before — // while every check passes, because the file is right and the service is up. That is not // hypothetical: it is how a third node joining a mesh left the first two carrying a network // that no longer existed, and every part of it reported success. // // This is declared state rather than a command. The declaration says the running service must // reflect these files; the host works out that it does not and acts. A *command* to restart // would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a // way around that rule, it is the shape the rule leaves. RestartOn []string `json:"restart-on,omitempty"` // ReloadOn names resources whose change means this service must be reloaded — for a service // that re-reads its configuration when told to, where a restart would stop what it runs: the // container runtime, whose restart stops every container on the machine (novox/hq ADR 0102). // A change that is also in RestartOn restarts it, which covers a reload. ReloadOn []string `json:"reload-on,omitempty"` } func (s *Service) Identity() string { return s.ID } func (s *Service) Kind() Type { return TypeService } func (s *Service) Target() string { return s.Unit } func (s *Service) validate(where string, _ bool) []string { var problems []string if s.Unit == "" { problems = append(problems, where+": a service needs a unit") } if s.State != "running" && s.State != "stopped" { problems = append(problems, fmt.Sprintf( "%s: state %q; a service is \"running\" or \"stopped\"", where, s.State)) } if s.Boot != "" && s.Boot != "enabled" && s.Boot != "disabled" { problems = append(problems, fmt.Sprintf( "%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+ "leave the machine's own setting alone", where, s.Boot)) } return problems } // IntoJSON is the one structured format a file is written into. const IntoJSON = "json" // Opening is a port reachable on an adopted node, from where, and on which path. // // **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming, // for something listening on the machine, or forwarded, for a published container port: the found // firewall sees a published port after the runtime has translated it, so a forwarded opening names // the container's own port in To as well as the machine's in Port. type Opening struct { ID string `json:"id"` Type Type `json:"type"` Port int `json:"port"` Protocol string `json:"protocol"` From string `json:"from"` Path string `json:"path"` To int `json:"to,omitempty"` } // Where an opening admits from, and the path it is on. const ( FromEverywhere = "everywhere" FromMesh = "mesh" PathIncoming = "incoming" PathForwarded = "forwarded" ) func (o *Opening) Identity() string { return o.ID } func (o *Opening) Kind() Type { return TypeOpening } func (o *Opening) Target() string { if o.Path == PathForwarded { return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From) } return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From) } func (o *Opening) validate(where string, _ bool) []string { var problems []string if o.Port < 1 || o.Port > 65535 { problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port)) } if o.Protocol != "tcp" && o.Protocol != "udp" { problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol)) } if o.From != FromEverywhere && o.From != FromMesh { problems = append(problems, fmt.Sprintf( "%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From)) } switch o.Path { case PathIncoming: if o.To != 0 { problems = append(problems, where+ ": an incoming opening names no container port; only a forwarded one does") } case PathForwarded: if o.To < 1 || o.To > 65535 { problems = append(problems, where+ ": a forwarded opening names the container's port it reaches, as to, 1-65535") } default: problems = append(problems, fmt.Sprintf( "%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path)) } if !strings.HasPrefix(o.ID, AdoptionPrefix) { problems = append(problems, fmt.Sprintf( "%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix)) } return problems } // Package is a package that should be present. // // Present is the whole of what it asserts, never a version: version is the package manager's // business and the mesh does not hold a second opinion about it. type Package struct { ID string `json:"id"` Type Type `json:"type"` Package string `json:"package"` } func (p *Package) Identity() string { return p.ID } func (p *Package) Kind() Type { return TypePackage } func (p *Package) Target() string { return p.Package } func (p *Package) validate(where string, _ bool) []string { if p.Package == "" { return []string{where + ": a package needs a package name"} } return nil } // Container is a container that should be running, from an image pinned by digest. type Container struct { ID string `json:"id"` Type Type `json:"type"` Name string `json:"name"` // Image is pinned by digest (novox/hq ADR 0006) — a tag moves and a digest does not. Image string `json:"image"` Env map[string]string `json:"env,omitempty"` // EnvFile names files the runtime reads environment from, in order. // // **Because a secret may not travel in Env.** A declaration reaches a node over the broker, // and `env` is plain text in it — so a password there is a password the broker sees, which is // the transitive trust refused everywhere else (novox/hq ADR 0004). A sealed file reaches the // machine unreadable, the host writes it, and the runtime reads it: the mesh never holds it // and neither does anything between them. // // It is also simply how third-party software takes credentials. Nothing that ships in a // container will read a path the mesh invented; every one of them reads its environment. EnvFile []string `json:"env-file,omitempty"` Ports []string `json:"ports,omitempty"` Volumes []string `json:"volumes,omitempty"` Args []string `json:"args,omitempty"` // Names this container can reach, as `name:address`. // // **Because a container does not inherit the machine's names.** It gets its own `/etc/hosts` // holding only its own hostname, so every internal name the mesh wrote for this machine is // invisible to the thing the machine is running. That was hit for real: a database client on // one node could not resolve another node, on a mesh where both names were correct and // present on both machines. // // **A file rather than a resolver, which is the decision the mesh already made about names** // and this extends rather than overturns: it works on every runtime, needs no package, and // has no failure mode of its own. A resolver becomes necessary when names are wanted that are // not one-per-node — service names, wildcards — and that is still not true. // // Set by the mesh, not by a module: which machines exist is a fact about the mesh, and a // module that listed them would be a module that goes stale when one joins. Hosts []string `json:"hosts,omitempty"` // Network is the container's network, passed to the runtime unchanged. // // Needed because the control plane must reach the store and the broker on the machine it was // raised on, before there is any mesh to arrange that. The alternative was publishing ports // and guessing an address that works from inside a container, which is the same thing with a // worse failure mode. Network string `json:"network,omitempty"` // RestartOn names resources whose change means this container must be recreated — the same // field a service has, for the same reason (novox/hq 04-ISSUES/009). A container reads a // mounted file once at start; a changed file leaves the running process holding the old value, // while every check passes because the file on disk is right. The container's spec — image, // env, volumes — does not include a mounted file's *content*, so a settings change that // re-renders that file is invisible to the ordinary spec diff. This closes that: the host // recreates the container when one of these resources changed this pass, even if the spec // matches. On a run-once step it means *run again*: a step that fetches a fact from a provider // names the binding it reads, and is run again when the provider moved (novox/hq ADR 0099). RestartOn []string `json:"restart-on,omitempty"` // RunOnce marks a container the host runs to completion rather than leaves running: a step, // not a service (novox/hq ADR 0052). The host runs it, requires it to exit 0, and records that // it did — and because the declaration is applied in order and a failed step halts the apply, // whatever is declared after a run-once container starts only once the step has finished. It is // how a module runs its own code at first boot — seed a store, migrate, health-gate — under its // own account (ADR 0047), before the container that depends on it. The record that it ran is // the digest of this declaration, so a re-apply does not re-run it unless the declaration // changed. RunOnce bool `json:"run-once,omitempty"` // Schedule marks a container the host runs on a recurring cadence — a five-field cron // expression (novox/hq ADR 0053). It is the recurring twin of RunOnce: the same container, run // to completion, but again and again on the clock rather than once. Installing it does not run // it — the schedule is state that is present, like a running service, so the apply is current as // soon as it is recorded and does NOT gate what follows. The host's scheduler fires the // container when the cron is due, re-established from this declaration each apply because the // declaration is the source of truth (ADR 0018). A run that exits non-zero is recorded and never // fails the apply or flips the node's state; a run still going when the next is due is skipped // rather than stacked. It is exclusive with RunOnce and with restart-on: a container runs once // and gates, runs on a cadence, or stays up — never two of these. Schedule string `json:"schedule,omitempty"` } func (c *Container) Identity() string { return c.ID } func (c *Container) Kind() Type { return TypeContainer } func (c *Container) Target() string { return c.Name } func (c *Container) validate(where string, _ bool) []string { var problems []string if c.Name == "" { problems = append(problems, where+": a container needs a name") } // A run-once step may name what it reads under restart-on. For a step the word means *run // again*: what a container reads is part of its digest, so a step whose named resource changed // is a different step and runs again (novox/hq ADR 0099). Not refused. // A container runs once and gates, on a cadence, or stays up — never two of these // (novox/hq ADR 0053). run-once and schedule are the two "runs to completion" lifecycles and // contradict each other, and a scheduled step does not stay running to be brought back by // restart-on either. Refused here on arrival, as the control plane refuses it near its author. if c.RunOnce && c.Schedule != "" { problems = append(problems, where+": a container is run-once or scheduled, not both; "+ "run-once runs once and gates what follows, a schedule runs it again on a cadence") } if c.Schedule != "" && len(c.RestartOn) > 0 { problems = append(problems, where+": a scheduled container cannot also declare restart-on; "+ "it runs to completion on its cadence rather than staying running to be restarted") } if c.Schedule != "" { if _, err := ParseCron(c.Schedule); err != nil { problems = append(problems, where+": "+err.Error()) } } return append(problems, checkImage(where, c.Image)...) } // Action runs something the bundle declared, and the host never learns what it means. type Action struct { ID string `json:"id"` Type Type `json:"type"` Command []string `json:"command"` // Verify is not optional and is not a courtesy. It is the read-back AND the idempotency // check: the host does not know what a database is, so "is it already there" is a question // only the declaration can ask (novox/hq ADR 0005). Verify []string `json:"verify"` // In names a container to run inside. Empty means the machine itself. In string `json:"in,omitempty"` } func (a *Action) Identity() string { return a.ID } func (a *Action) Kind() Type { return TypeAction } func (a *Action) Target() string { target := strings.Join(a.Command, " ") if a.In != "" { return "in " + a.In + ": " + target } return target } func (a *Action) validate(where string, allowActions bool) []string { // The bound the whole security argument rests on (novox/hq ADR 0005). if !allowActions { return []string{where + ": an action arrived over the link, and the link may not carry one. The host " + "applies declarations of known shape; a command to run is not one. A bundle may " + "carry an action because it arrives with the binary — anyone able to put a " + "hostile action there could have put it in the host itself"} } var problems []string if len(a.Command) == 0 { problems = append(problems, where+": an action needs a command") } if len(a.Verify) == 0 { problems = append(problems, where+ ": an action needs a verify. An action that runs and reports success without "+ "reading anything back is the fault this host exists to prevent, and verify is "+ "also how the host knows whether the action is already done") } return problems } // newOf returns an empty resource of a kind, or nil if the kind is unknown. // // This is the whole vocabulary, in one place. A kind that is not here cannot be declared. func newOf(t Type) Resource { switch t { case TypeDirectory: return &Directory{} case TypeFile: return &File{} case TypeService: return &Service{} case TypePackage: return &Package{} case TypeContainer: return &Container{} case TypeAction: return &Action{} case TypeNetwork: return &Network{} case TypeUser: return &User{} case TypeArchive: return &Archive{} case TypeAccess: return &Access{} case TypeProcess: return &Process{} case TypeOpening: return &Opening{} } return nil } // Vocabulary is every kind this host speaks. func Vocabulary() []Type { return []Type{ TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile, TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser, } } // Declaration is what a machine should be, in the order it should be made so. type Declaration struct { Version int // For names the node this is meant for. A host with an identity refuses one addressed // elsewhere; a host without one — the first node, applying the bundle it carries — has // nothing to check against. For string // Resources, in the order they are applied. The host does not sort them: ordering is a // decision, and deciding is not what the host does (novox/hq ADR 0005). Resources []Resource // Adoption says this node is adopted, and which of its modules have been taken. Nil is a // converged node — which is every node the mesh raised before adoption existed, and so the // only form an older controller ever sends (novox/hq ADR 0100). Adoption *Adoption } // Adoption is a node's mode, as the controller records it: the node is adopted, and these are // the modules taken on it so far (novox/hq ADR 0100). // // **Authoritative, and only ever stated by the controller.** A host does not work out whether it // is adopted; it is told, in every declaration, so a host restarted from the declaration it kept // is in the same mode it was in before. // // Untaken names, per module assigned here and not yet taken, the ids of its file and container // resources — the only shapes a predecessor can already have on the machine. The host cannot // split a resource id into its module, because module names may contain dots, so the controller // says which ids belong to which module rather than leaving the host to guess. type Adoption struct { Taken []string `json:"taken"` Untaken map[string][]string `json:"untaken,omitempty"` } // AdoptionPrefix is the id prefix of what the mesh itself declares because a node is adopted — // its openings and its guard. Nothing under it belongs to a module, so none of it is ever held. const AdoptionPrefix = "adoption." // UntakenModuleOf says which untaken module declares a resource, if any. func (a *Adoption) UntakenModuleOf(id string) (string, bool) { if a == nil { return "", false } for module, ids := range a.Untaken { if slices.Contains(ids, id) { return module, true } } return "", false } // checkAdoption holds what an adoption says against the resources beside it. Every problem is a // refusal: a host that misread which module is untaken would replace a predecessor's service the // operator never took. func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []string { if a == nil { return nil } if allowActions { // The bundle is carried with the binary and raises a foundation before any mesh exists. // Whether a node is adopted is the controller's record, and a bundle that claimed it would // be the host deciding its own mode (novox/hq ADR 0100). return []string{"a carried bundle says the node is adopted, and only the mesh can say " + "that: a node's mode is the controller's record, sent in every declaration"} } kinds := map[string]Type{} for _, r := range resources { kinds[r.Identity()] = r.Kind() } var problems []string for _, module := range a.Taken { if _, both := a.Untaken[module]; both { problems = append(problems, fmt.Sprintf( "adoption: the module %q is said to be both taken and untaken", module)) } } owner := map[string]string{} modules := make([]string, 0, len(a.Untaken)) for module := range a.Untaken { modules = append(modules, module) } sort.Strings(modules) for _, module := range modules { for _, id := range a.Untaken[module] { if strings.HasPrefix(id, AdoptionPrefix) { problems = append(problems, fmt.Sprintf( "adoption: %q is the mesh's own and belongs to no module, so it cannot be untaken", id)) continue } if first, twice := owner[id]; twice { problems = append(problems, fmt.Sprintf( "adoption: %q is said to belong to both %q and %q", id, first, module)) continue } owner[id] = module kind, declared := kinds[id] switch { case !declared: problems = append(problems, fmt.Sprintf( "adoption: %q of the untaken module %q is not in this declaration", id, module)) case kind != TypeFile && kind != TypeContainer: problems = append(problems, fmt.Sprintf( "adoption: %q of the untaken module %q is a %s, and only a file or a "+ "container can be found on a machine", id, module, kind)) } } } return problems } // RefusalError refuses a whole declaration, naming every problem at once. // // Every problem rather than the first: a caller fixing one at a time learns the next only by // running again, and a declaration is generated, so a person reading this is debugging the // generator. type RefusalError struct { Problems []string } func (e *RefusalError) Error() string { return fmt.Sprintf( "this declaration is refused, and none of it was applied:\n - %s\n\n"+ "A host that applied the parts it understood would leave a machine that looks "+ "configured and is not.", strings.Join(e.Problems, "\n - ")) } // Parse reads a declaration that arrived over the link, and refuses anything it does not fully // understand — including any action, which the link may not carry (novox/hq ADR 0005). func Parse(raw []byte) (*Declaration, error) { return parse(raw, false) } // ParseTrusted reads a declaration from a source already as privileged as the host itself: the // bundle it carries, or a file handed to it by someone who is running it as root. // // Actions are permitted here and nowhere else. The asymmetry is deliberate and is the entire // content of ADR 0005: refusing actions from the bundle buys nothing, because whoever built the // bundle built the binary; refusing them from the link buys the bound on what a compromised // control plane can express. func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) } // envelope is the declaration with its resources still unread. // // Two passes, because which fields are legal depends on the "type" inside each resource. The // first pass takes the envelope and each resource's bytes; the second decodes each one into // the struct for its kind, strictly. type envelope struct { Version int `json:"declaration"` For string `json:"for,omitempty"` Adoption *Adoption `json:"adoption,omitempty"` Resources []json.RawMessage `json:"resources"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { var env envelope if err := strictDecode(raw, &env); err != nil { return nil, &RefusalError{Problems: []string{"not a declaration: " + err.Error()}} } if env.Version != Version { // Everything below assumes the vocabulary, so there is nothing further to say. return nil, &RefusalError{Problems: []string{fmt.Sprintf( "declaration version %d; this host speaks version %d. Refused whole rather than "+ "partly, so a newer vocabulary is never half-applied by an older host", env.Version, Version)}} } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption} var problems []string if len(env.Resources) == 0 { problems = append(problems, "no resources. An empty declaration is a mistake, not a "+ "machine with nothing on it — say so with an explicit empty list if that is meant") } seen := map[string]int{} for i, rawResource := range env.Resources { // Peek, leniently. This pass only needs to know which struct to decode into; reading // strictly here would report an unknown field before knowing which fields are known. var head struct { ID string `json:"id"` Type Type `json:"type"` } _ = json.Unmarshal(rawResource, &head) where := fmt.Sprintf("resource %d", i) if head.ID != "" { where = fmt.Sprintf("resource %q", head.ID) } if head.ID == "" { problems = append(problems, where+": no id. Identity is what lets the host know "+ "this is the same resource it applied last time") } else if first, ok := seen[head.ID]; ok { problems = append(problems, fmt.Sprintf( "%s: id already used by resource %d. Two resources with one identity cannot "+ "both be tracked", where, first)) } else { seen[head.ID] = i } resource := newOf(head.Type) if resource == nil { problems = append(problems, fmt.Sprintf( "%s: unknown type %q. This host understands %s", where, head.Type, vocabulary())) continue } // A field the kind does not have is refused, and the struct is what says so — there // is no list of exclusions for anyone to keep current. // // Asked separately rather than taken from the decoder's error, because the decoder // stops at the first unknown field and this record promises every problem at once. A // caller fixing one field at a time learns the next only by running again. if unknown := unknownFields(rawResource, resource); len(unknown) > 0 { for _, field := range unknown { problems = append(problems, fmt.Sprintf( "%s: a %s does not use %q, and it is set. Refused rather than ignored", where, head.Type, field)) } continue } if err := json.Unmarshal(rawResource, resource); err != nil { problems = append(problems, fmt.Sprintf("%s: %s", where, err)) continue } problems = append(problems, resource.validate(where, allowActions)...) d.Resources = append(d.Resources, resource) } problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) if env.Adoption == nil { for _, r := range d.Resources { if r.Kind() == TypeOpening { // On a converged node the mesh's own filter admits what is declared, and the // found firewall is retired; an opening there would be a rule in a firewall the // mesh has disabled (novox/hq ADR 0100). problems = append(problems, fmt.Sprintf( "resource %q: an opening is for an adopted node, and this declaration does not "+ "say the node is adopted", r.Identity())) } } } if len(problems) > 0 { return nil, &RefusalError{Problems: problems} } return d, nil } func strictDecode(raw []byte, into any) error { // DisallowUnknownFields is the whole point rather than strictness for its own sake: a // field the host does not know is a thing the control plane believes it asked for. dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(into); err != nil { return err } // And **nothing after it**. A decoder reads one value and stops, so a file holding a // declaration followed by anything at all — a truncated rewrite, two declarations // concatenated, a stray line from whatever wrote the file — parses as the first value and the // rest is never looked at. // // That is the same fault this host refuses everywhere else, in its quietest form: the machine // applies something, reports success, and what it applied is not what the file says. Found // when a test harness appended a line to a bundle by accident and every apply kept working. if _, err := dec.Token(); err != io.EOF { return fmt.Errorf( "there is more in this file after the declaration ends. Refused whole: a file with " + "something after it may be a truncated rewrite or two declarations run together, " + "and applying the first would be applying something nobody wrote") } return nil } // unknownFields names every JSON key the kind's struct has no field for. // // The struct's own tags are the list of what is legal, so adding a field to a kind is the // whole of adding it — there is nowhere else that has to agree. func unknownFields(raw []byte, into Resource) []string { var got map[string]json.RawMessage if err := json.Unmarshal(raw, &got); err != nil { return nil // not an object; the decode below will say so properly } known := map[string]bool{} t := reflect.TypeOf(into).Elem() for i := 0; i < t.NumField(); i++ { name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") if name != "" && name != "-" { known[name] = true } } var unknown []string for field := range got { if !known[field] { unknown = append(unknown, field) } } sort.Strings(unknown) return unknown } func checkMode(where, mode string) []string { if mode == "" { return nil } if len(mode) != 4 || mode[0] != '0' { return []string{fmt.Sprintf( "%s: mode %q; write it as four octal digits such as \"0644\", so it means the "+ "same thing here as it does in the manifest it came from", where, mode)} } for _, c := range mode[1:] { if c < '0' || c > '7' { return []string{fmt.Sprintf("%s: mode %q is not octal", where, mode)} } } return nil } // checkImage insists on content, not on a name. // // A tag moves and a digest does not. The bundle's whole claim is that what it names is exact // (novox/hq ADR 0006), and a bundle pinning `postgres:17` pins nothing — it names whatever // that tag points at on the day the host happens to run. // // **Two forms say something exact, and only one of them needs a registry.** `name@sha256:…` is a // manifest digest, which a registry assigns on push. A bare `sha256:…` is an image the machine // already holds, addressed by the digest of its own configuration — equally immutable, equally // unforgeable, and requiring nothing to have served it. // // That second form is what a first machine needs. The mesh's own control plane exists in no public // registry and never will: it is built from source, and until this mesh has a registry of its own // there is nowhere to push it to and therefore no manifest digest to name it by. Insisting on one // would mean a registry has to exist before the thing that lets a mesh have a registry can start — // which is not a pin, it is a dependency the rule accidentally created. A machine that built an // image, or was handed one, can name it by what it is. func checkImage(where, image string) []string { if image == "" { return []string{where + ": a container needs an image"} } // An image this machine holds, named by the digest of its own configuration. if strings.HasPrefix(image, "sha256:") { if len(image) != len("sha256:")+64 { return []string{fmt.Sprintf( "%s: image id %q is not a sha256 digest", where, image)} } return nil } name, digest, found := strings.Cut(image, "@") if !found || name == "" { return []string{fmt.Sprintf( "%s: image %q is not pinned. Write it as name@sha256:… — or as sha256:… for an image "+ "this machine already holds. A tag moves, and a bundle that pinned a tag would "+ "not be pinned", where, image)} } if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { return []string{fmt.Sprintf( "%s: image digest %q is not a sha256 digest", where, digest)} } return nil } func vocabulary() string { kinds := Vocabulary() names := make([]string, 0, len(kinds)) for _, t := range kinds { names = append(names, string(t)) } sort.Strings(names) return strings.Join(names, ", ") } // ParseFileTrusted reads a declaration from a file somebody handed this host. // // The same as ParseTrusted, and it allows whole-line `//` comments first. A pinned, hand-authored // artefact that nobody can annotate is one nobody can review — the foundation bundle is mostly // explanation of why each digest is what it is. // // **Only for a file, never for the link.** Over the link the format stays exactly JSON, because // a wire format with a second thing to strip is a wire format with a second thing to disagree // about. // // It exists because there were two readers for one file: the bundle stripped comments and `apply` // did not, so the example bundle in this repository could be built into a binary and not applied // from disk. The failure was `invalid character '/'`, which names the symptom and not the cause. func ParseFileTrusted(raw []byte) (*Declaration, error) { return ParseTrusted(stripComments(raw)) } // stripComments removes whole lines beginning with `//`. // // Only whole lines: anything cleverer would need to know where strings begin and end, and a // parser that half-understands its input is worse than one that does not try. A `//` inside a // value — every image reference has one — is untouched. func stripComments(raw []byte) []byte { var kept []string for _, line := range strings.Split(string(raw), "\n") { if strings.HasPrefix(strings.TrimSpace(line), "//") { continue } kept = append(kept, line) } return []byte(strings.Join(kept, "\n")) }