package delivery import ( "encoding/json" "fmt" "path" "sort" "strconv" "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) // Placement of modules on nodes lives in the catalogue (novox/hq ADR 0549): one file per node, `placement/.json` // in the catalogue repository, changed only by a pull request. At each trunk merge of the catalogue the indexer reads // the files and writes each node's key `placement.` in its state `placement`, only when it differs, and says // `placement-changed` naming the node, the revision and the modules added, removed or changed (ADR 0549 §2). The node // agents, the assigner, the grants part and mesh-config read it (§3); nothing in it is secret. const ( // PlacementState is the indexer's state of placements, and PlacementBucket its bucket. PlacementState = "placement" PlacementBucket = IndexerModule + "_" + PlacementState // PlacementHistory is the revisions the placement bucket keeps per key. PlacementHistory = 10 // PlacementDir is the catalogue's directory of placement files, one `.json` per node. PlacementDir = "placement" // PlacementChangedEvent is what the indexer says when a node's placement key changed. PlacementChangedEvent = "placement-changed" placementWord = "placement" // CatalogueRepository is the repository whose placement files are the mesh's placement (ADR 0549 §1): the forge bridge // reads them at each of its merges, and no other repository's. CatalogueRepository = "novox/mesh-catalog" ) // PlacementKey is the key, in PlacementBucket, of one node's placement: `placement.`. func PlacementKey(node string) (string, error) { if err := checkName("node", node); err != nil { return "", err } return placementWord + "." + node, nil } // ParsePlacementKey is the inverse of PlacementKey; it refuses any other key, naming it. func ParsePlacementKey(key string) (node string, err error) { node, ok := strings.CutPrefix(key, placementWord+".") if !ok || checkName("node", node) != nil { return "", fmt.Errorf("delivery: %q is not a placement key, placement.", key) } return node, nil } // PlacementFile is the path, in the catalogue repository, of one node's placement file. func PlacementFile(node string) string { return path.Join(PlacementDir, node+".json") } // PlacementFileNode is the node a catalogue path is the placement file of, and whether it is one: `placement/.json` // directly under PlacementDir. func PlacementFileNode(p string) (string, bool) { dir, file := path.Split(p) node, ok := strings.CutSuffix(file, ".json") if path.Clean(dir) != PlacementDir || !ok || checkName("node", node) != nil { return "", false } return node, true } // Provider is the module on a node that answers a requirement or holds a seat: a pin's or a binding's provider. type Provider struct { Node string `json:"node"` Module string `json:"module"` } // Chosen is p as the composition names a chosen provider. func (p Provider) Chosen() catalogue.Chosen { return catalogue.Chosen{Node: p.Node, Module: p.Module} } // PlacedModule is one module placed on the node: the host port each port it declares is published on (by the port the // software listens on), and, by provision, the provider each of its requirements that keeps data is bound to. // // MeshWide and Layer are where the module's things are, as a settings layer's placement holds them (catalogue.Layer's // Placement: `places`, `accesses`, `networks`, `endpoints`, `expose`, `reach`, `ports`; ADR 0348 §11): the whole mesh's // layer and this node's, which the controller keeps in its table `placement` (ADR 0549 §6 names it) and composes as two // layers, the mesh's first. Each node's file repeats the mesh-wide layer of the modules placed on it, so a node reads // its own key alone. HasLayer says the node has a layer of its own even when it holds no placement, since a layer // that is there and empty still carries the module's config values (Layers). type PlacedModule struct { Module string `json:"module"` Ports map[string]int `json:"ports,omitempty"` Bindings map[string]Provider `json:"bindings,omitempty"` MeshWide map[string]any `json:"mesh-wide,omitempty"` HasMesh bool `json:"has-mesh-wide,omitempty"` Layer map[string]any `json:"layer,omitempty"` HasLayer bool `json:"has-layer,omitempty"` } // Holding is one seat a module placed on the node holds (catalogue.Held without its node, which is the file's). type Holding struct { Claim string `json:"claim"` Scope string `json:"scope,omitempty"` Module string `json:"module"` Site string `json:"site,omitempty"` Kind string `json:"kind,omitempty"` } // Placement is one node's placement: the file the catalogue keeps and, with Revision, the value of its key. Modules are // in the order the file names them, which is the order the composition reads the node's modules in. Pins is, by // provision, the provider chosen for the node where more than one could answer. type Placement struct { V int `json:"v"` Node string `json:"node"` Revision string `json:"revision,omitempty"` Decided NodeDecided `json:"decided"` Modules []PlacedModule `json:"modules"` Pins map[string]Provider `json:"pins,omitempty"` Holds []Holding `json:"holds,omitempty"` } // NodeDecided is what a person decides of the node, kept in its placement file (issue 1126's K2: what a person decides // lives in the catalogue, what the machine observes lives in its record, NodeRecord): where it is dialled (endpoint) and // its site, whether it is the private network's hub, its address on the private network, the domain its routed names // are composed under, its operator and agent logins by name and their homes, whether it is adopted rather than // converged, the modules taken on an adopted node, the tunnel found on it at adoption and, on the hub, the names given // to that tunnel's peers. The address is a decision written once, at enrolment, by a // catalogue pull request: nothing allocates it at run time, and CheckPlacements refuses two nodes with one address. // No secret and no key material: accounts are names, never credentials. type NodeDecided struct { Endpoint string `json:"endpoint,omitempty"` Site string `json:"site,omitempty"` Hub bool `json:"hub,omitempty"` Address string `json:"address,omitempty"` PublicDomain string `json:"public-domain,omitempty"` Account string `json:"account,omitempty"` AccountHome string `json:"account-home,omitempty"` AgentAccount string `json:"agent-account,omitempty"` AgentHome string `json:"agent-account-home,omitempty"` Adopted bool `json:"adopted,omitempty"` Taken []string `json:"taken,omitempty"` // Tunnel is the tunnel found on the node when it was adopted, kept as decided then: once the node takes it over it is // no longer there to observe. PeerNames is, on the hub, the name a person gave a peer of that tunnel, by its public key. Tunnel *RecordTunnel `json:"tunnel,omitempty"` PeerNames map[string]string `json:"peer-names,omitempty"` } func (p Placement) check() error { if err := checkName("node", p.Node); err != nil { return err } seen := map[string]bool{} for _, m := range p.Modules { if err := checkName("module", m.Module); err != nil { return err } if seen[m.Module] { return fmt.Errorf("delivery: the placement of %s names %s twice", p.Node, m.Module) } seen[m.Module] = true for port, at := range m.Ports { if n, err := strconv.Atoi(port); err != nil || n <= 0 || n > 65535 { return fmt.Errorf("delivery: %s's port %q on %s is not a port number", m.Module, port, p.Node) } if at <= 0 || at > 65535 { return fmt.Errorf("delivery: %s's port %s on %s is published on %d, which is not a port number", m.Module, port, p.Node, at) } } for provision, to := range m.Bindings { if provision == "" || to.Node == "" || to.Module == "" { return fmt.Errorf("delivery: %s on %s has a binding with no provision, node or module", m.Module, p.Node) } } } for provision, to := range p.Pins { if provision == "" || to.Node == "" || to.Module == "" { return fmt.Errorf("delivery: the placement of %s has a pin with no provision, node or module", p.Node) } } for _, h := range p.Holds { if h.Claim == "" || !seen[h.Module] { return fmt.Errorf("delivery: the placement of %s has %s hold %q, which is not a module placed there or names no seat", p.Node, h.Module, h.Claim) } } return nil } // EncodePlacement is p as JSON at Version; it refuses what DecodePlacement refuses. func EncodePlacement(p Placement) ([]byte, error) { if err := p.check(); err != nil { return nil, err } p.V = Version return json.Marshal(p) } // DecodePlacement reads a placement file or key value; it refuses an unknown version (ErrUnknownVersion), a node or // module that is not a name, a module named twice, a port that is not a number, a binding or pin naming no provider, // and a holding by a module not placed on the node. func DecodePlacement(data []byte) (Placement, error) { var p Placement if err := decode(data, &p); err != nil { return Placement{}, err } if err := p.check(); err != nil { return Placement{}, err } return p, nil } // SamePlacement says whether two placements are the same value, so the indexer writes only when they differ. The // revision is not compared: a merge that changes no placement writes nothing (ADR 0549 §2). func SamePlacement(a, b Placement) bool { a.Revision, b.Revision = "", "" ja, errA := EncodePlacement(a) jb, errB := EncodePlacement(b) return errA == nil && errB == nil && string(ja) == string(jb) } // PlacementChange is what `placement-changed` says: the node, the catalogue revision, and its modules added, removed // or changed against the placement before. type PlacementChange struct { Node string `json:"node"` Revision string `json:"revision"` Added []string `json:"added,omitempty"` Removed []string `json:"removed,omitempty"` Changed []string `json:"changed,omitempty"` } // ChangeOf is what changed from before to after on one node; before is the zero Placement for a node new to placement. // A change of the node's pins or holdings counts as a change of every module placed on it both times, since the // composition of each may read them. func ChangeOf(before, after Placement) PlacementChange { c := PlacementChange{Node: after.Node, Revision: after.Revision} if c.Node == "" { c.Node = before.Node } was := map[string]PlacedModule{} for _, m := range before.Modules { was[m.Module] = m } nodeWide := !jsonEqual(before.Pins, after.Pins) || !jsonEqual(before.Holds, after.Holds) now := map[string]bool{} for _, m := range after.Modules { now[m.Module] = true old, ok := was[m.Module] switch { case !ok: c.Added = append(c.Added, m.Module) case nodeWide || !jsonEqual(old, m): c.Changed = append(c.Changed, m.Module) } } for _, m := range before.Modules { if !now[m.Module] { c.Removed = append(c.Removed, m.Module) } } sort.Strings(c.Added) sort.Strings(c.Removed) sort.Strings(c.Changed) return c } func jsonEqual(a, b any) bool { ja, _ := json.Marshal(a) jb, _ := json.Marshal(b) return string(ja) == string(jb) } // Assigned is the modules placed on the node, in the file's order (compose.Node's Assigned). func (p Placement) Assigned() []string { out := make([]string, 0, len(p.Modules)) for _, m := range p.Modules { out = append(out, m.Module) } return out } // ChosenPins is the node's pins as the composition reads them (compose.Node's Pins); nil when it has none. func (p Placement) ChosenPins() map[string]catalogue.Chosen { if len(p.Pins) == 0 { return nil } out := make(map[string]catalogue.Chosen, len(p.Pins)) for provision, to := range p.Pins { out[provision] = to.Chosen() } return out } // Bound is, by consumer module and provision, where each binding sends it (compose.Node's Bound); nil when none. func (p Placement) Bound() map[string]map[string]catalogue.Chosen { var out map[string]map[string]catalogue.Chosen for _, m := range p.Modules { if len(m.Bindings) == 0 { continue } if out == nil { out = map[string]map[string]catalogue.Chosen{} } out[m.Module] = make(map[string]catalogue.Chosen, len(m.Bindings)) for provision, to := range m.Bindings { out[m.Module][provision] = to.Chosen() } } return out } // HostPorts is, by module and the port it declares, the host port it is published on (compose.Node's Ports); nil when // none. The check has refused a port that is not a number, so none is dropped here. func (p Placement) HostPorts() map[string]map[int]int { var out map[string]map[int]int for _, m := range p.Modules { if len(m.Ports) == 0 { continue } if out == nil { out = map[string]map[int]int{} } out[m.Module] = make(map[int]int, len(m.Ports)) for port, at := range m.Ports { n, _ := strconv.Atoi(port) out[m.Module][n] = at } } return out } // Layers is one module's settings layers on the node, as the composition reads them (compose.Node's Settings): the // whole mesh's layer when it has one, then the node's, which carries the module's config values as mesh-config resolved // them for this node (values, nil when none) beside its placement; a module with no layer and no value has none. This is // the controller's SettingsFor for a module that takes its config values from mesh-config (issue 664). func (m PlacedModule) Layers(node string, values map[string]any) []catalogue.Layer { var out []catalogue.Layer if m.HasMesh || len(m.MeshWide) > 0 { out = append(out, catalogue.Layer{From: catalogue.MeshWideLayer, Values: map[string]any{}, Placement: m.MeshWide}) } if m.HasLayer || len(m.Layer) > 0 || len(values) > 0 { if values == nil { values = map[string]any{} } out = append(out, catalogue.Layer{From: node, Values: values, Placement: m.Layer}) } return out } // Held is the node's holdings as the composition reads them (compose.Input's Holdings rows for this node). func (p Placement) Held() []catalogue.Held { out := make([]catalogue.Held, 0, len(p.Holds)) for _, h := range p.Holds { out = append(out, catalogue.Held{Claim: h.Claim, Scope: h.Scope, Node: p.Node, Module: h.Module, Site: h.Site, Kind: h.Kind}) } return out } // CheckPlacements is the catalogue's own check of every placement file (ADR 0549 §1), over the placements by node, the // modules the catalogue holds and the nodes enrolled. It returns every refusal, in node order, never only the first: a // placement naming a node not enrolled or a module the catalogue does not hold; two modules of one node on the same host // port; a pin, binding or holding naming a provider that is not placed where it says; a seat of scope mesh held twice; // two nodes with one address on the private network; more than one hub. // A host port a manifest declares without a number and a requirement left unbound are the composition's to find (its // ForeseenPorts and Unbound), which the catalogue's composition check reads. func CheckPlacements(placements map[string]Placement, held func(module string) bool, enrolled func(node string) bool) []string { var refusals []string nodes := make([]string, 0, len(placements)) for n := range placements { nodes = append(nodes, n) } sort.Strings(nodes) placed := func(p Provider) bool { other, ok := placements[p.Node] if !ok { return false } for _, m := range other.Modules { if m.Module == p.Module { return true } } return false } meshSeats := map[string]string{} addresses := map[string]string{} var hubs []string for _, node := range nodes { p := placements[node] if p.Node != node { refusals = append(refusals, fmt.Sprintf("the placement file of %s names the node %q", node, p.Node)) } if a := strings.TrimSpace(p.Decided.Address); a != "" { if other, twice := addresses[a]; twice { refusals = append(refusals, fmt.Sprintf("%s and %s both have the address %s on the private network", other, node, a)) } addresses[a] = node } if p.Decided.Hub { hubs = append(hubs, node) } if !enrolled(node) { refusals = append(refusals, fmt.Sprintf("%s is not an enrolled node", node)) } hostPorts := map[int]string{} for _, m := range p.Modules { if !held(m.Module) { refusals = append(refusals, fmt.Sprintf("%s: the catalogue holds no module %s", node, m.Module)) } for _, port := range sortedKeys(m.Ports) { at := m.Ports[port] if other, clash := hostPorts[at]; clash && other != m.Module { refusals = append(refusals, fmt.Sprintf("%s: %s and %s are both published on host port %d", node, other, m.Module, at)) } hostPorts[at] = m.Module } for _, provision := range sortedKeys(m.Bindings) { if to := m.Bindings[provision]; !placed(to) { refusals = append(refusals, fmt.Sprintf("%s: %s's %s is bound to %s/%s, which is not placed there", node, m.Module, provision, to.Node, to.Module)) } } } for _, provision := range sortedKeys(p.Pins) { if to := p.Pins[provision]; !placed(to) { refusals = append(refusals, fmt.Sprintf("%s: the pin of %s names %s/%s, which is not placed there", node, provision, to.Node, to.Module)) } } for _, h := range p.Holds { if h.Scope != "mesh" { continue } seat := h.Claim + "/" + h.Kind if other, twice := meshSeats[seat]; twice { refusals = append(refusals, fmt.Sprintf("the seat %s, held once for the mesh, is held on %s and %s", h.Claim, other, node)) } meshSeats[seat] = node } } if len(hubs) > 1 { refusals = append(refusals, fmt.Sprintf("the private network has one hub, and %s are each placed as it", strings.Join(hubs, ", "))) } return refusals } func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out }