package delivery import ( "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "regexp" "sort" "strings" "time" ) // A placement changes only by a proposal (novox/hq ADR 0567 §2): `mesh-placement.propose` takes why and a list of // changes; the holder lays them over the current keys, names the revision of each key it laid them over, and either // refuses at once with every finding or writes the proposal under its id in its state `proposals` and a candidate for // each node it must hear from (PlacementCandidate). The proposal's key leaves the state when it is applied, withdrawn, // declined or expired; its whole content stays in the event that ends it (PlacementEvent), so no proposal is kept in // the state once it has ended. const ( // PlacementProposalsState is the seat's state of open proposals, PlacementProposalsBucket its bucket and // PlacementProposalsRead the name a reader's manifest declares; a key is the proposal's id. PlacementProposalsState = "proposals" PlacementProposalsBucket = AssignerModule + "_" + PlacementProposalsState PlacementProposalsRead = AssignerModule + "." + PlacementProposalsState // PlacementProposalsHistory is the revisions the bucket keeps per key. PlacementProposalsHistory = 5 // PlacementPulsesState is the holder's state of bounds (§4): a key per node a proposal waits on, written with a time // to live of the answer bound, and a key per raised ask, with its ask's; the server's delete marker when one ends is // the event that fails a silent node or ends an unanswered ask, so the holder keeps no timer. Its bucket allows a // time to live per key (catalogue.StateDeclaration.KeyTTL). PlacementPulsesState = "pulses" PlacementPulsesBucket = AssignerModule + "_" + PlacementPulsesState ) // PulseAnswerKey is the key of the bound on one node's answer to a proposal: `answer..`. func PulseAnswerKey(proposal, node string) (string, error) { if err := CheckProposalID(proposal); err != nil { return "", err } if err := checkName("node", node); err != nil { return "", err } return "answer." + proposal + "." + node, nil } // PulseAskKey is the key of the bound on a proposal's raised ask: `ask.`. func PulseAskKey(proposal string) (string, error) { if err := CheckProposalID(proposal); err != nil { return "", err } return "ask." + proposal, nil } // The operations a proposal's change names (ADR 0567 §2). const ( OpPlace = "place" // place a module on a node, with its host ports and bindings OpTakeOff = "take-off" // take a module off a node OpPin = "pin" // pin the provider of a provision on a node OpUnpin = "unpin" // remove a pin OpHold = "hold" // a module placed on the node holds a seat OpUnhold = "unhold" // it holds it no more OpDecide = "decide" // set a decided field of the node (NodeDecided by its JSON name), its address among them OpConfig = "config" // set a module's config placement or values on the node OpSeed = "seed" // the node's whole placement: the take-over (adopt, §7), never given by a caller ) // The states of an open proposal. const ( // ProposalAnswering: its candidates are written and the nodes it must hear from have not all answered. ProposalAnswering = "answering" // ProposalFailed: its verdict failed; it stays open, for its proposer to read and withdraw, or for a node's later // answer to judge it again. ProposalFailed = "failed" // ProposalAsked: its verdict passed and the operator is asked to approve it. ProposalAsked = "asked" ) // VerdictWaits is a proposal's verdict while a node it must hear from has not answered within the bound; it passes // (VerdictPass) or fails (VerdictFail) as a check's verdict does (ADR 0567 §4). const VerdictWaits = "waits" // PlacementProposalChange is one change of a proposal. Node is always named; the rest is the operation's. type PlacementProposalChange struct { Op string `json:"op"` Node string `json:"node"` Module string `json:"module,omitempty"` Ports map[string]int `json:"ports,omitempty"` Bindings map[string]Provider `json:"bindings,omitempty"` Provision string `json:"provision,omitempty"` Provider *Provider `json:"provider,omitempty"` // Claim, Scope, Kind and Site are a holding's (OpHold, OpUnhold). Claim string `json:"claim,omitempty"` Scope string `json:"scope,omitempty"` Kind string `json:"kind,omitempty"` Site string `json:"site,omitempty"` // Field and Value are a decided field's (OpDecide): its JSON name in NodeDecided and its value. Field string `json:"field,omitempty"` Value json.RawMessage `json:"value,omitempty"` // MeshWide, Layer, MeshValues and Values are a module's config placement and values (OpConfig), as PlacedModule // carries them; each given replaces what the module has. MeshWide map[string]any `json:"mesh-wide,omitempty"` Layer map[string]any `json:"layer,omitempty"` MeshValues map[string]any `json:"mesh-wide-values,omitempty"` Values map[string]any `json:"values,omitempty"` // Whole is the node's whole placement (OpSeed). Whole *Placement `json:"whole,omitempty"` } // BeforeLease is why the change cannot be proposed before the lease write, or "" (ADR 0567 §8): until then the holder // keeps the controller's assignments in step through its verbs assign, unassign, pin and unpin, so only those are // proposable — a module placed or taken off with no host port or binding, a pin set or removed, a module's config values // and the seed. A host port, a binding, a holding, a decided field and a config layer's places or accesses wait for the // lease write, which the refusal names. func (c PlacementProposalChange) BeforeLease() string { const lease = "which the controller's verbs cannot take from a caller, so it waits for the lease write (ADR 0567 §8)" switch c.Op { case OpPlace: if len(c.Ports)+len(c.Bindings) > 0 { return fmt.Sprintf("the place of %s on %s names host ports or bindings, %s", c.Module, c.Node, lease) } case OpTakeOff, OpPin, OpUnpin, OpSeed: case OpConfig: // A module's config values are the state's alone (§13); its layers' places and accesses are the controller's // until the lease write. if c.MeshWide != nil || c.Layer != nil { return fmt.Sprintf("the config of %s on %s sets a layer's places or accesses, %s", c.Module, c.Node, lease) } default: return fmt.Sprintf("a change %s of %s is no assignment or pin, %s", c.Op, c.Node, lease) } return "" } func (c PlacementProposalChange) check() error { if err := checkName("node", c.Node); err != nil { return err } needs := func(what string, ok bool) error { if !ok { return fmt.Errorf("delivery: a %s change of %s names no %s", c.Op, c.Node, what) } return nil } switch c.Op { case OpPlace, OpTakeOff, OpConfig: return needs("module", checkName("module", c.Module) == nil) case OpPin: return needs("provision or provider", c.Provision != "" && c.Provider != nil && c.Provider.Node != "" && c.Provider.Module != "") case OpUnpin: return needs("provision", c.Provision != "") case OpHold, OpUnhold: return needs("module or claim", checkName("module", c.Module) == nil && c.Claim != "") case OpDecide: return needs("field or value", c.Field != "" && len(c.Value) > 0) case OpSeed: if c.Whole == nil || c.Whole.Node != c.Node { return fmt.Errorf("delivery: the seed of %s names no placement of that node", c.Node) } return c.Whole.check() } return fmt.Errorf("delivery: %q is no change of placement: place, take-off, pin, unpin, hold, unhold, decide or config", c.Op) } // LayChanges is the placements of the nodes the changes name with the changes laid over current, in the order given; // current is every node's placement now (absent: the node has no key). It returns, by node, the laid placement, and // every finding: a change that does not fit its node (a module taken off that is not placed, configured or holding // there; a field NodeDecided does not have) or a placement DecodePlacement refuses. What CheckPlacements refuses // across nodes is the caller's to ask of the result laid over current. func LayChanges(current map[string]Placement, changes []PlacementProposalChange) (map[string]Placement, []string) { laid := map[string]Placement{} var findings []string of := func(node string) Placement { if p, ok := laid[node]; ok { return p } p, ok := current[node] if !ok { return Placement{V: Version, Node: node, Modules: []PlacedModule{}} } return clonePlacement(p) } for i, c := range changes { if err := c.check(); err != nil { findings = append(findings, fmt.Sprintf("change %d: %v", i+1, err)) continue } p := of(c.Node) at := -1 for j, m := range p.Modules { if m.Module == c.Module { at = j } } notPlaced := func() { findings = append(findings, fmt.Sprintf("change %d: %s is not placed on %s", i+1, c.Module, c.Node)) } switch c.Op { case OpSeed: p = clonePlacement(*c.Whole) case OpPlace: m := PlacedModule{Module: c.Module, Ports: c.Ports, Bindings: c.Bindings} if at >= 0 { m.MeshWide, m.HasMesh, m.Layer, m.HasLayer = p.Modules[at].MeshWide, p.Modules[at].HasMesh, p.Modules[at].Layer, p.Modules[at].HasLayer m.MeshValues, m.Values = p.Modules[at].MeshValues, p.Modules[at].Values p.Modules[at] = m } else { p.Modules = append(p.Modules, m) } case OpTakeOff: if at < 0 { notPlaced() continue } p.Modules = append(p.Modules[:at:at], p.Modules[at+1:]...) var holds []Holding for _, h := range p.Holds { if h.Module != c.Module { holds = append(holds, h) } } p.Holds = holds case OpConfig: if at < 0 { notPlaced() continue } m := &p.Modules[at] if c.MeshWide != nil { m.MeshWide, m.HasMesh = c.MeshWide, true } if c.Layer != nil { m.Layer, m.HasLayer = c.Layer, true } if c.MeshValues != nil { m.MeshValues = c.MeshValues } if c.Values != nil { m.Values = c.Values } case OpPin: if p.Pins == nil { p.Pins = map[string]Provider{} } p.Pins[c.Provision] = *c.Provider case OpUnpin: delete(p.Pins, c.Provision) if len(p.Pins) == 0 { p.Pins = nil } case OpHold: if at < 0 { notPlaced() continue } p.Holds = append(p.Holds, Holding{Claim: c.Claim, Scope: c.Scope, Module: c.Module, Site: c.Site, Kind: c.Kind}) case OpUnhold: kept := p.Holds[:0:0] for _, h := range p.Holds { if h.Module != c.Module || h.Claim != c.Claim { kept = append(kept, h) } } if len(kept) == len(p.Holds) { findings = append(findings, fmt.Sprintf("change %d: %s holds no %s on %s", i+1, c.Module, c.Claim, c.Node)) continue } p.Holds = kept case OpDecide: d, err := decide(p.Decided, c.Field, c.Value) if err != nil { findings = append(findings, fmt.Sprintf("change %d: %v", i+1, err)) continue } p.Decided = d } laid[c.Node] = p } for _, node := range sortedKeys(laid) { if err := laid[node].check(); err != nil { findings = append(findings, err.Error()) } } return laid, findings } // decide is d with the field named by its JSON name set to value, refused when NodeDecided has no such field or the // value does not fit it. func decide(d NodeDecided, field string, value json.RawMessage) (NodeDecided, error) { raw, _ := json.Marshal(d) var fields map[string]json.RawMessage _ = json.Unmarshal(raw, &fields) if fields == nil { fields = map[string]json.RawMessage{} } if !decidedFields[field] { return d, fmt.Errorf("a node has no decided field %q", field) } fields[field] = value raw, _ = json.Marshal(fields) var out NodeDecided dec := json.NewDecoder(strings.NewReader(string(raw))) dec.DisallowUnknownFields() if err := dec.Decode(&out); err != nil { return d, fmt.Errorf("the decided field %q does not take %s: %v", field, value, err) } return out, nil } // decidedFields are NodeDecided's fields by their JSON names. var decidedFields = func() map[string]bool { raw, _ := json.Marshal(NodeDecided{Endpoint: "x", Site: "x", Hub: true, Address: "x", PublicDomain: "x", Account: "x", AccountHome: "x", AgentAccount: "x", AgentHome: "x", Adopted: true, Taken: []string{"x"}, Tunnel: &RecordTunnel{}, PeerNames: map[string]string{"x": "x"}}) var m map[string]any _ = json.Unmarshal(raw, &m) out := map[string]bool{} for k := range m { out[k] = true } return out }() func clonePlacement(p Placement) Placement { raw, _ := json.Marshal(p) var out Placement _ = json.Unmarshal(raw, &out) if out.Modules == nil { out.Modules = []PlacedModule{} } return out } var proposalIDOK = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{2,62}$`) // CheckProposalID refuses an id that is no proposal's: lower-case letters, digits and dashes, 3 to 63 of them, so it is // one key token. func CheckProposalID(id string) error { if !proposalIDOK.MatchString(id) { return fmt.Errorf("delivery: %q is no proposal id (lower-case letters, digits and dashes, one key token)", id) } return nil } // NewProposalID is a fresh proposal id: its time in UTC and six random hex digits, `p-20261028t183000-1a2b3c`. func NewProposalID(at time.Time) string { var b [3]byte _, _ = rand.Read(b[:]) return "p-" + strings.ToLower(at.UTC().Format("20060102T150405")) + "-" + hex.EncodeToString(b[:]) } // PlacementVerdict is the holder's judgement of a proposal's answers (ADR 0567 §4). It fails only on a refusal by a // node that must answer or on its silence past the bound (and, for the seed alone, on any difference); every difference // is shown by node: Intended is what the holder computed the proposal changes, Unintended what a node answered beside // it, marked as such in the brief. type PlacementVerdict struct { State string `json:"state"` At time.Time `json:"at"` Fails []string `json:"fails,omitempty"` Silent []string `json:"silent,omitempty"` Until time.Time `json:"until,omitempty"` Intended map[string][]string `json:"intended,omitempty"` Unintended map[string][]string `json:"unintended,omitempty"` // Carried is, for the seed, by node each difference a node found that the seed carries, named for the operator who // approves it knowingly (": at "): the seed fails on no difference, only on a refusal or // silence (issue 1126). Carried map[string][]string `json:"carried,omitempty"` Answers map[string]CandidateAnswer `json:"answers,omitempty"` Notes []string `json:"notes,omitempty"` } // PlacementProposal is an open proposal, the value of its key in PlacementProposalsBucket. type PlacementProposal struct { V int `json:"v"` ID string `json:"id"` By string `json:"by"` Why string `json:"why"` // At is when it was proposed, Written when its newest candidate was written (the answers' bound runs from it). At time.Time `json:"at"` Written time.Time `json:"written,omitempty"` // Changes are as proposed; Seed says they are the take-over (adopt, §7), ExportDigest the digest of the export it // proposed (ExportDigest), checked again at the warrant. Changes []PlacementProposalChange `json:"changes"` Seed bool `json:"seed,omitempty"` ExportDigest string `json:"export-digest,omitempty"` // Base is, by node whose key it changes, the revision of the key it was laid over (0: the node had none). Base map[string]uint64 `json:"base"` // Must is every node it must hear from (§3): each whose key it changes and each whose composition it changes // through another's, sorted; set by the holder once it composed, before then the nodes of Base. Must []string `json:"must,omitempty"` // HandBack says it is the operator's hand-back (§7): every key of Base deleted on its warrant, with no change and no // node's answer. HandBack bool `json:"hand-back,omitempty"` // State is ProposalAnswering, ProposalFailed or ProposalAsked; Verdict is the latest judgement, Ask the authorising // ask raised on a passing one. State string `json:"state"` Verdict *PlacementVerdict `json:"verdict,omitempty"` Ask string `json:"ask,omitempty"` // Words is what happened last, in words (laid again over moved keys, asked again). Words string `json:"words,omitempty"` // Again is when `again` asked the holder to judge it now and raise its ask again (§10); the holder does once per // value. Again time.Time `json:"again,omitempty"` // Withdrawn is who withdrew it and why (`withdraw`, §2): the holder then deletes it and its candidates and says // placement-withdrawn, which only the holder's process can say on its stream. Withdrawn string `json:"withdrawn,omitempty"` } // Nodes is every node whose key the proposal changes, sorted. func (p PlacementProposal) Nodes() []string { return sortedKeys(p.Base) } func (p PlacementProposal) check() error { if err := CheckProposalID(p.ID); err != nil { return err } if strings.TrimSpace(p.By) == "" || strings.TrimSpace(p.Why) == "" { return fmt.Errorf("delivery: the proposal %s names no proposer or no why", p.ID) } if len(p.Base) == 0 || (len(p.Changes) == 0) != p.HandBack || (p.HandBack && p.Seed) { return fmt.Errorf("delivery: the proposal %s changes nothing, or is a hand-back with changes", p.ID) } for i, c := range p.Changes { if err := c.check(); err != nil { return fmt.Errorf("delivery: the proposal %s, change %d: %w", p.ID, i+1, err) } if (c.Op == OpSeed) != p.Seed { return fmt.Errorf("delivery: the proposal %s mixes the seed with other changes", p.ID) } if _, ok := p.Base[c.Node]; !ok { return fmt.Errorf("delivery: the proposal %s changes %s and names no base revision of its key", p.ID, c.Node) } } switch p.State { case ProposalAnswering, ProposalFailed: case ProposalAsked: if p.Ask == "" || p.Verdict == nil || p.Verdict.State != VerdictPass { return fmt.Errorf("delivery: the proposal %s is asked with no ask or no passing verdict", p.ID) } default: return fmt.Errorf("delivery: the proposal %s is in no state %q", p.ID, p.State) } if p.Seed && p.ExportDigest == "" { return fmt.Errorf("delivery: the seed %s names no export digest", p.ID) } return nil } // EncodePlacementProposal is p as JSON at Version; it refuses what DecodePlacementProposal refuses. func EncodePlacementProposal(p PlacementProposal) ([]byte, error) { if err := p.check(); err != nil { return nil, err } p.V = Version return json.Marshal(p) } // DecodePlacementProposal reads a proposal; it refuses an unknown version, an id that is none, no proposer or why, no // change, a change it does not take or of a node with no base revision, the seed mixed with other changes or without its // export digest, and a state it does not know. func DecodePlacementProposal(data []byte) (PlacementProposal, error) { var p PlacementProposal if err := decode(data, &p); err != nil { return PlacementProposal{}, err } if err := p.check(); err != nil { return PlacementProposal{}, err } return p, nil } // ExportDigest is the digest of an export's answer, every node's placement, as the seed proposes it: the same // placements give the same digest, whatever order they came in. func ExportDigest(placements map[string]Placement) string { h := sha256.New() for _, node := range sortedKeys(placements) { raw, _ := EncodePlacement(placements[node]) fmt.Fprintf(h, "%s\n%s\n", node, raw) } return "sha256:" + hex.EncodeToString(h.Sum(nil)) } // ChangesDigest is the digest of a proposal's changes, which its authorising ask is bound to (§5). func ChangesDigest(p PlacementProposal) string { raw, _ := json.Marshal(struct { ID string `json:"id"` Changes []PlacementProposalChange `json:"changes"` Base map[string]uint64 `json:"base"` }{p.ID, p.Changes, p.Base}) sum := sha256.Sum256(raw) return "sha256:" + hex.EncodeToString(sum[:]) } // PlacementWarrant is who approved a proposal, through which channel and when (§5, §6): the warrant's id as the teller // wrote it (the ask's id and when), and its words, which name the person and the channel that proved them // (asks.Warrant.Says). type PlacementWarrant struct { Warrant string `json:"warrant"` Words string `json:"words"` At time.Time `json:"at"` } // PlacementEvent is what each of the seat's events carries (§6): the proposal whole (its id, by, why, changes, base // revisions and verdict with every node's answer); for placement-changed, by node the revisions before and after and // the modules added, removed or changed, and the warrant; for placement-withdrawn, why it ended. type PlacementEvent struct { Proposal PlacementProposal `json:"proposal"` Nodes []PlacementChange `json:"nodes,omitempty"` Warrant *PlacementWarrant `json:"warrant,omitempty"` Ended string `json:"ended,omitempty"` } // SortChanges orders a placement-changed event's nodes by node. func SortChanges(c []PlacementChange) { sort.Slice(c, func(i, j int) bool { return c[i].Node < c[j].Node }) }