package delivery import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "sort" "strings" "sync" ) // **One key per module and node** (research 072 row 37, the operator's decision of 2026-10-31; issue 774). The seat // mesh-placement's state `placement` keeps a node's placement under the node's own keys, so a node watches `.>` // alone and a change of one module writes that module's key: // // - `.node`, the node key: what a person decided of the node (NodeDecided), its pins and its holdings, and the // modules placed on it in the order the composition reads them, each with the digest of its module key's value. // The word `node` is the have state's own word for a node's key (HaveNodeWord), so no module is named it. // - `.`, a module key: the module's ports, bindings and layers (PlacedModule). // // The node key is the commit of a write (ADR 0567 ยง6: a proposal's keys share one revision in history). A writer writes // the module keys first, each carrying the value it replaces as Previous, then the node key, by compare-and-set at the // revision the proposal was laid over, and deletes the module keys no longer named last. A reader takes, for each module // the node key names, the value of its module key whose digest the node key names: the current one once the node key // is written, the previous one while the write is under way. So a reader never composes a part of a write, and every // write of a node is one revision of its node key, whose history is the node's placement history as `placement.`'s // was (Placed, the warrant's record). A module key the node key does not name is not placed. // // The form before, one value `placement.` per node (LegacyPlacementKey), is still read during the delivery of // this change, so readers and the holder may move in any order: a node with a node key is read from its new keys alone, // one without from its legacy key. The holder converts each legacy value once at its start (PlacementKeys.Migrations), // keeps the legacy key in step after every commit as a mirror, so a reader of the form before alone still follows every // write while the members of the delivery move in any order, and deletes it first when a node's placement is deleted, // so a placement handed back never returns from it. A later change stops the mirror and deletes `placement.*`. const ( // PlacementNodeWord is the last word of a node key, `.node`. PlacementNodeWord = HaveNodeWord placementDigestLen = 16 ) // PlacementKeyKind is the kind of a key of the state `placement`. type PlacementKeyKind int // The kinds of key: a node key, a module key, and the form before (one value per node). const ( PlacementNodeKind PlacementKeyKind = iota + 1 PlacementModuleKind PlacementLegacyKind ) // PlacementNodeKey is a node's node key, `.node`. A node named `placement` is refused: its keys would read as the // legacy form's. func PlacementNodeKey(node string) (string, error) { if err := checkPlacementNode(node); err != nil { return "", err } return node + "." + PlacementNodeWord, nil } // PlacementModuleKey is a module's key on a node, `.`; a module named `node` is refused. func PlacementModuleKey(node, module string) (string, error) { if err := checkPlacementNode(node); err != nil { return "", err } if err := checkPlacementModule(module); err != nil { return "", err } return node + "." + module, nil } // PlacementNodeKeys is the pattern of every key of one node, `.>`: what a node watches of its own placement. func PlacementNodeKeys(node string) (string, error) { if err := checkPlacementNode(node); err != nil { return "", err } return node + ".>", nil } // LegacyPlacementKey is a node's key in the form before, `placement.`, read while this change is delivered. func LegacyPlacementKey(node string) (string, error) { if err := checkName("node", node); err != nil { return "", err } return placementWord + "." + node, nil } // ParsePlacementKey reads a key of the state `placement` in either form: its node, its module (empty for a node key and // a legacy key) and its kind. Any other key is refused, naming it. func ParsePlacementKey(key string) (node, module string, kind PlacementKeyKind, err error) { first, rest, ok := strings.Cut(key, ".") if !ok || strings.Contains(rest, ".") || checkName("node", first) != nil || checkName("module", rest) != nil { return "", "", 0, fmt.Errorf("delivery: %q is not a placement key, .node or .", key) } switch { case first == placementWord: return rest, "", PlacementLegacyKind, nil case rest == PlacementNodeWord: return first, "", PlacementNodeKind, nil default: return first, rest, PlacementModuleKind, nil } } func checkPlacementNode(node string) error { if err := checkName("node", node); err != nil { return err } if node == placementWord { return fmt.Errorf("delivery: no node is named %q: its placement keys would read as the form before", placementWord) } return nil } func checkPlacementModule(module string) error { if err := checkName("module", module); err != nil { return err } if module == PlacementNodeWord { return fmt.Errorf("delivery: no module is named %q: it is the word of a node's own key", PlacementNodeWord) } return nil } // PlacementRef is one module the node key names, with the digest of the module key's value it stands for. type PlacementRef struct { Module string `json:"module"` Digest string `json:"digest"` } // PlacementNode is the value of a node key: the node's decisions, pins and holdings, and the modules placed on it. type PlacementNode struct { V int `json:"v"` Node string `json:"node"` Decided NodeDecided `json:"decided"` Modules []PlacementRef `json:"modules"` Pins map[string]Provider `json:"pins,omitempty"` Holds []Holding `json:"holds,omitempty"` } // PlacementModule is the value of a module key: the module as placed, and, while a write that replaces it is under way, // the value it replaces (Previous), which a reader takes until the node key names the new one. type PlacementModule struct { V int `json:"v"` Node string `json:"node"` Placed PlacedModule `json:"placed"` Previous *PlacedModule `json:"previous,omitempty"` } // Digest is the digest of a placed module's value, as a node key names it. func (m PlacedModule) Digest() string { raw, _ := json.Marshal(m) sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:])[:placementDigestLen] } // EncodePlacementNode is n as JSON at Version. func EncodePlacementNode(n PlacementNode) ([]byte, error) { if err := checkPlacementNode(n.Node); err != nil { return nil, err } n.V = Version return json.Marshal(n) } // DecodePlacementNode reads a node key's value; it refuses an unknown version, a node or module that is not a name, and // a module named twice or without a digest. func DecodePlacementNode(data []byte) (PlacementNode, error) { var n PlacementNode if err := decode(data, &n); err != nil { return PlacementNode{}, err } if err := checkPlacementNode(n.Node); err != nil { return PlacementNode{}, err } seen := map[string]bool{} for _, r := range n.Modules { if err := checkPlacementModule(r.Module); err != nil { return PlacementNode{}, err } if seen[r.Module] || r.Digest == "" { return PlacementNode{}, fmt.Errorf("delivery: the node key of %s names %s twice or without a digest", n.Node, r.Module) } seen[r.Module] = true } return n, nil } // EncodePlacementModule is m as JSON at Version. func EncodePlacementModule(m PlacementModule) ([]byte, error) { if err := checkPlacementNode(m.Node); err != nil { return nil, err } if err := checkPlacementModule(m.Placed.Module); err != nil { return nil, err } if m.Previous != nil && m.Previous.Module != m.Placed.Module { return nil, fmt.Errorf("delivery: the key of %s on %s carries %s as its previous value", m.Placed.Module, m.Node, m.Previous.Module) } m.V = Version return json.Marshal(m) } // DecodePlacementModule reads a module key's value. func DecodePlacementModule(data []byte) (PlacementModule, error) { var m PlacementModule if err := decode(data, &m); err != nil { return PlacementModule{}, err } if err := checkPlacementNode(m.Node); err != nil { return PlacementModule{}, err } if err := checkPlacementModule(m.Placed.Module); err != nil { return PlacementModule{}, err } return m, nil } // PlacementKeys is the state `placement` as a reader holds it, key by key in either form, and every node's placement // assembled from it: the one reader of the state's keys for every part (issue 774), so no part parses them itself. A // write of the holder's under way is never assembled half: a module key's value is taken only as far as the node key // names its digest (see the package's comment above PlacementNodeWord), and a node whose keys do not agree keeps the // placement last assembled for it, Put saying why. // // Safe for use from a watch and from a reader at once. type PlacementKeys struct { mu sync.Mutex nodes map[string]*placementKeysOf } // placementKeysOf is what is held of one node's keys. type placementKeysOf struct { node *PlacementNode nodeRev uint64 modules map[string]PlacementModule moduleRevs map[string]uint64 legacy *Placement legacyRev uint64 last *Placement // the placement last assembled whole, nil when none } // NewPlacementKeys holds no key yet. func NewPlacementKeys() *PlacementKeys { return &PlacementKeys{nodes: map[string]*placementKeysOf{}} } // PlacementsOf is every node's placement assembled from the state's keys as a reader holds them (key to value; a // deleted key absent), and an error naming every key that did not read and every node whose keys do not agree (that // node is then assembled from its module keys' values as they are). func PlacementsOf(keys map[string][]byte) (map[string]Placement, error) { k := NewPlacementKeys() var errs []error names := make([]string, 0, len(keys)) for key := range keys { names = append(names, key) } sort.Strings(names) for _, key := range names { // A node key read before its module keys does not agree yet; whether the keys agree is asked once all are in. if _, readErr, _ := k.put(key, keys[key], 0, false); readErr != nil { errs = append(errs, readErr) } } k.mu.Lock() for _, node := range sortedKeys(k.nodes) { if _, _, err := k.nodes[node].assemble(node); err != nil { errs = append(errs, err) } } k.mu.Unlock() return k.All(), errors.Join(errs...) } // Put takes one key's value in (revision: the bus's revision of it, 0 when the reader has none; deleted: a delete or a // purge). It answers the node the key is of and an error for a key or a value that does not read, or for a node whose // keys do not agree; the node then keeps what it had. func (k *PlacementKeys) Put(key string, value []byte, revision uint64, deleted bool) (string, error) { node, readErr, err := k.put(key, value, revision, deleted) if readErr != nil { return node, readErr } return node, err } // put is Put telling an error of the key or value it took (readErr) from the node's keys not agreeing after it (err). func (k *PlacementKeys) put(key string, value []byte, revision uint64, deleted bool) (node string, readErr, err error) { node, module, kind, err := ParsePlacementKey(key) if err != nil { return "", err, nil } k.mu.Lock() defer k.mu.Unlock() of := k.nodes[node] if of == nil { of = &placementKeysOf{modules: map[string]PlacementModule{}, moduleRevs: map[string]uint64{}} k.nodes[node] = of } switch kind { case PlacementNodeKind: of.node, of.nodeRev = nil, 0 if !deleted { n, derr := DecodePlacementNode(value) switch { case derr != nil: readErr = fmt.Errorf("the node key %s does not read, so it places nothing there: %w", key, derr) case n.Node != node: readErr = fmt.Errorf("the node key %s names the node %q, so it places nothing there", key, n.Node) default: of.node, of.nodeRev = &n, revision } } case PlacementModuleKind: delete(of.modules, module) delete(of.moduleRevs, module) if !deleted { m, derr := DecodePlacementModule(value) switch { case derr != nil: readErr = fmt.Errorf("the module key %s does not read: %w", key, derr) case m.Node != node || m.Placed.Module != module: readErr = fmt.Errorf("the module key %s holds %s on %s", key, m.Placed.Module, m.Node) default: of.modules[module] = m } // Its revision is kept even when its value does not read, so the holder can write over it. of.moduleRevs[module] = revision } case PlacementLegacyKind: of.legacy, of.legacyRev = nil, 0 if !deleted { p, derr := DecodePlacement(value) switch { case derr != nil: readErr = fmt.Errorf("the placement of %s does not read, so it places nothing there: %w", node, derr) case p.Node != node: readErr = fmt.Errorf("the placement under %s names the node %q, so it places nothing there", key, p.Node) default: of.legacy, of.legacyRev = &p, revision } } } p, whole, err := of.assemble(node) switch { case p == nil: of.last = nil case whole || of.last == nil: of.last = p } return node, readErr, err } // assemble is the node's placement from its keys: nil when none places it, whole false (and an error) when the node key // names a module digest no module key's value has. func (of *placementKeysOf) assemble(node string) (*Placement, bool, error) { if of.node == nil { if of.legacy == nil { return nil, true, nil } p := clonePlacement(*of.legacy) return &p, true, nil } n := of.node p := Placement{V: Version, Node: node, Decided: n.Decided, Pins: n.Pins, Holds: n.Holds, Modules: make([]PlacedModule, 0, len(n.Modules))} var astray []string for _, r := range n.Modules { m, ok := of.modules[r.Module] switch { case ok && m.Placed.Digest() == r.Digest: p.Modules = append(p.Modules, m.Placed) case ok && m.Previous != nil && m.Previous.Digest() == r.Digest: p.Modules = append(p.Modules, *m.Previous) case ok: astray = append(astray, r.Module) p.Modules = append(p.Modules, m.Placed) default: astray = append(astray, r.Module) } } if len(astray) > 0 { return &p, false, fmt.Errorf("the node key of %s names %s at a value no module key holds: %s keeps the placement it had", node, strings.Join(astray, ", "), node) } return &p, true, nil } // Of is node's placement and the revision of the key it was read from (its node key, or its legacy key), and whether // any key places it. func (k *PlacementKeys) Of(node string) (Placement, uint64, bool) { k.mu.Lock() defer k.mu.Unlock() of := k.nodes[node] if of == nil || of.last == nil { return Placement{}, 0, false } rev := of.nodeRev if of.node == nil { rev = of.legacyRev } return clonePlacement(*of.last), rev, true } // All is every node's placement, by node, and the revision of each one's key (Of). func (k *PlacementKeys) All() map[string]Placement { out, _ := k.AllRevisions() return out } // AllRevisions is All with the revision of each node's key. func (k *PlacementKeys) AllRevisions() (map[string]Placement, map[string]uint64) { k.mu.Lock() nodes := make([]string, 0, len(k.nodes)) for n := range k.nodes { nodes = append(nodes, n) } k.mu.Unlock() out, revs := map[string]Placement{}, map[string]uint64{} for _, n := range nodes { if p, rev, ok := k.Of(n); ok { out[n], revs[n] = p, rev } } return out, revs } // PlacementKeyWrite is one write of a key of the state `placement`, by compare-and-set: Value nil deletes the key at // Base, Base 0 creates it, any other Base updates it at that revision. type PlacementKeyWrite struct { Node string Key string Value []byte Base uint64 // Commit says this write is the node key's: once it is written, the node's placement is the new one. Commit bool // Mirror says this write is the legacy key's, `placement.`, kept in step after the commit for the readers not // yet delivered: a reader of both forms never reads it once the node key is there, so a mirror write that fails // harms no such reader, and is made again at the node's next write. Mirror bool } // Writes are the writes that take node from the placement its keys hold to after (nil: no placement), in the order // they are made: the module keys that change, each carrying the value it replaces; then the node key, the commit; then // the legacy key `placement.` with the whole placement, a mirror for the readers not yet delivered, unless it // holds it already; then the deletes of the module keys after no longer names. For no placement the legacy key goes // first, then the node key, then the module keys, so a crash between them never leaves the legacy value to be read // again. A module key whose value stays is not written. func (k *PlacementKeys) Writes(node string, after *Placement) ([]PlacementKeyWrite, error) { nodeKey, err := PlacementNodeKey(node) if err != nil { return nil, err } k.mu.Lock() defer k.mu.Unlock() of := k.nodes[node] if of == nil { of = &placementKeysOf{modules: map[string]PlacementModule{}, moduleRevs: map[string]uint64{}} } var out []PlacementKeyWrite deleteModules := func(keep map[string]bool) { for _, module := range sortedKeys(of.moduleRevs) { if keep[module] { continue } key, _ := PlacementModuleKey(node, module) out = append(out, PlacementKeyWrite{Node: node, Key: key, Base: of.moduleRevs[module]}) } } if after == nil { if of.legacy != nil { key, _ := LegacyPlacementKey(node) out = append(out, PlacementKeyWrite{Node: node, Key: key, Base: of.legacyRev}) } if of.node != nil { out = append(out, PlacementKeyWrite{Node: node, Key: nodeKey, Base: of.nodeRev, Commit: true}) } deleteModules(nil) return out, nil } if after.Node != node { return nil, fmt.Errorf("delivery: the placement written under %s names the node %q", node, after.Node) } if err := after.check(); err != nil { return nil, err } committed := map[string]PlacedModule{} if of.last != nil { for _, m := range of.last.Modules { committed[m.Module] = m } } n := PlacementNode{Node: node, Decided: after.Decided, Pins: after.Pins, Holds: after.Holds, Modules: make([]PlacementRef, 0, len(after.Modules))} keep := map[string]bool{} for _, m := range after.Modules { if err := checkPlacementModule(m.Module); err != nil { return nil, err } digest := m.Digest() n.Modules = append(n.Modules, PlacementRef{Module: m.Module, Digest: digest}) keep[m.Module] = true was, wasPlaced := committed[m.Module] if held, ok := of.modules[m.Module]; ok && held.Placed.Digest() == digest && (held.Previous == nil || wasPlaced && was.Digest() == digest) { continue // the key holds it, and no previous value a reader could take in its place } v := PlacementModule{Node: node, Placed: m} if wasPlaced && was.Digest() != digest { v.Previous = &was } raw, err := EncodePlacementModule(v) if err != nil { return nil, err } key, _ := PlacementModuleKey(node, m.Module) out = append(out, PlacementKeyWrite{Node: node, Key: key, Value: raw, Base: of.moduleRevs[m.Module]}) } raw, err := EncodePlacementNode(n) if err != nil { return nil, err } out = append(out, PlacementKeyWrite{Node: node, Key: nodeKey, Value: raw, Base: of.nodeRev, Commit: true}) // The mirror of the form before, while readers of it alone may run (the delivery of this change); a later change // stops writing it and deletes it. if of.legacy == nil || !SamePlacement(*of.legacy, *after) { legacy, err := EncodePlacement(*after) if err != nil { return nil, err } key, _ := LegacyPlacementKey(node) out = append(out, PlacementKeyWrite{Node: node, Key: key, Value: legacy, Base: of.legacyRev, Mirror: true}) } deleteModules(keep) return out, nil } // Migrations are, by node, the writes that convert every node held only in the form before (its legacy key, no node // key) into its node and module keys, its legacy key kept as the mirror for the readers not yet delivered (Writes); empty // once every node is converted, so the holder's conversion at each start is idempotent. func (k *PlacementKeys) Migrations() (map[string][]PlacementKeyWrite, error) { k.mu.Lock() var legacy []string for node, of := range k.nodes { if of.node == nil && of.legacy != nil && checkPlacementNode(node) == nil { legacy = append(legacy, node) } } k.mu.Unlock() sort.Strings(legacy) out := map[string][]PlacementKeyWrite{} var errs []error for _, node := range legacy { p, _, _ := k.Of(node) w, err := k.Writes(node, &p) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", node, err)) continue } out[node] = w } return out, errors.Join(errs...) } // Legacy says node is held only in the form before. func (k *PlacementKeys) Legacy(node string) bool { k.mu.Lock() defer k.mu.Unlock() of := k.nodes[node] return of != nil && of.node == nil && of.legacy != nil } // PlacementValues is every key and value of p written whole onto no keys before: its module keys, its node key and the // legacy mirror, as a writer that holds no key of the node (a test's bus, a fresh mesh) writes them. func PlacementValues(p Placement) (map[string][]byte, error) { writes, err := NewPlacementKeys().Writes(p.Node, &p) if err != nil { return nil, err } out := make(map[string][]byte, len(writes)) for _, w := range writes { out[w.Key] = w.Value } return out, nil } // InsideAddresses is every node's address on the private network as its placement decides it (NodeDecided.Address), // sorted and each once: the machines inside the private network, which a membership's Mesh carried until the assignments // stream went (ADR 0167; research 072 rows 33 and 34). A node with no address decided is left out. func InsideAddresses(placements map[string]Placement) []string { seen := map[string]bool{} out := []string{} for _, p := range placements { if a := strings.TrimSpace(p.Decided.Address); a != "" && !seen[a] { seen[a] = true out = append(out, a) } } sort.Strings(out) return out }