From 146cd858a309d56aaf53c1a05b2c2c96ad28e7bd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:02:13 +0100 Subject: [PATCH 01/42] =?UTF-8?q?Placement=20and=20the=20node=20record=20h?= =?UTF-8?q?ave=20one=20layout=20in=20mesh-sdk,=20so=20the=20indexer,=20the?= =?UTF-8?q?=20node=20agents=20and=20the=20composition=20check=20read=20the?= =?UTF-8?q?m=20alike=20(issue=201126,=20ADR=200549=20=C2=A71-=C2=A72,=20AD?= =?UTF-8?q?R=200547=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/delivery_test.go | 6 +- go/delivery/noderecord.go | 156 ++++++++++++++ go/delivery/noderecord_test.go | 79 +++++++ go/delivery/placement.go | 371 +++++++++++++++++++++++++++++++++ go/delivery/placement_test.go | 152 ++++++++++++++ 5 files changed, 762 insertions(+), 2 deletions(-) create mode 100644 go/delivery/noderecord.go create mode 100644 go/delivery/noderecord_test.go create mode 100644 go/delivery/placement.go create mode 100644 go/delivery/placement_test.go diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index 564e6ad..99cfeba 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -14,6 +14,8 @@ import ( // 1131/AC1: the have key forms and parses back; bad names are refused, naming them. func TestHaveKey(t *testing.T) { + t.Run("presence", testPresenceKey) + t.Run("pulse", testPulseKey) if HaveBucket != "node-engine_have" { t.Fatalf("HaveBucket = %q", HaveBucket) } @@ -58,7 +60,7 @@ func TestHaveKey(t *testing.T) { } // 1131/AC2: the presence keys form and parse back; a node named part is refused. -func TestPresenceKey(t *testing.T) { +func testPresenceKey(t *testing.T) { if PresenceBucket != "mesh-delivery-watchman_presence" { t.Fatalf("PresenceBucket = %q", PresenceBucket) } @@ -98,7 +100,7 @@ func TestPresenceKey(t *testing.T) { } // 1131/AC3: the pulse key splits at the first dot, the key holding dots. -func TestPulseKey(t *testing.T) { +func testPulseKey(t *testing.T) { if PulseBucket != "mesh-delivery-watchman_pulse" { t.Fatalf("PulseBucket = %q", PulseBucket) } diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go new file mode 100644 index 0000000..72c8ac0 --- /dev/null +++ b/go/delivery/noderecord.go @@ -0,0 +1,156 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// A node's record is what the node knows of itself, written by its own agent into the node-engine's state `records`, +// one key `record.` per node, and read by every node and by the parts that compose (novox/hq ADR 0547 §2: "of +// other nodes it reads only what is public: their records"; issue 1126's prepare, L1 as narrowed by ADR 0549). Where a +// module runs, its host ports, pins, bindings and seat holdings are not in it: they are the catalogue's placement +// (placement.go). Nothing in it is secret: its keys are public halves, its accounts are login names. +// +// The agent writes it at start and whenever a fact it holds changes, only when the value differs, and says +// `node-record-changed`. Seeded from what the node runs and reports, never from the controller's store. +const ( + // NodeRecordsState is the node-engine's state of node records, and NodeRecordsBucket its bucket. + NodeRecordsState = "records" + NodeRecordsBucket = "node-engine_" + NodeRecordsState + // NodeRecordsHistory is the revisions the records bucket keeps per key. + NodeRecordsHistory = 5 + // NodeRecordChangedEvent is what a node's agent says when it wrote its record. + NodeRecordChangedEvent = "node-record-changed" + + recordWord = "record" +) + +// NodeRecordKey is the key, in NodeRecordsBucket, of one node's record: `record.`. +func NodeRecordKey(node string) (string, error) { + if err := checkName("node", node); err != nil { + return "", err + } + return recordWord + "." + node, nil +} + +// ParseNodeRecordKey is the inverse of NodeRecordKey; it refuses any other key, naming it. +func ParseNodeRecordKey(key string) (node string, err error) { + node, ok := strings.CutPrefix(key, recordWord+".") + if !ok || checkName("node", node) != nil { + return "", fmt.Errorf("delivery: %q is not a node record key, record.", key) + } + return node, nil +} + +// NodePlace is where the node is on the private network: its tunnel key (public), endpoint, site, whether it is the +// hub, and its address (compose.Place). +type NodePlace struct { + Key string `json:"key,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Site string `json:"site,omitempty"` + Hub bool `json:"hub,omitempty"` + Address string `json:"address,omitempty"` +} + +// RecordTunnel is a tunnel found on the node, as compose.Tunnel holds it: public keys and addresses only. +type RecordTunnel struct { + Interface string `json:"interface"` + Unit string `json:"unit,omitempty"` + Config string `json:"config,omitempty"` + Port int `json:"port,omitempty"` + MTU int `json:"mtu,omitempty"` + Address string `json:"address,omitempty"` + Range string `json:"range,omitempty"` + PublicKey string `json:"public-key,omitempty"` + Peers []RecordTunnelPeer `json:"peers,omitempty"` +} + +// RecordTunnelPeer is one peer of a found tunnel. +type RecordTunnelPeer struct { + PublicKey string `json:"public-key"` + Address string `json:"address,omitempty"` +} + +// RecordCarriedPeer is a peer of the tunnel the hub adopted, which the hub carries (compose.CarriedPeer). +type RecordCarriedPeer struct { + PublicKey string `json:"public-key"` + Address string `json:"address,omitempty"` + EnrolledAs string `json:"enrolled-as,omitempty"` + Named string `json:"named,omitempty"` +} + +// NodeRecord is one node's record. Tunnel is the tunnel found on it whose key is its private-network key; on the hub +// alone, Adopted says whether the hub took its found tunnel over and Carried is every peer of it the hub carries, so +// the tunnels and the carried peers are each one node's own and need no other home. +type NodeRecord struct { + V int `json:"v"` + Node string `json:"node"` + Place NodePlace `json:"place"` + Capabilities []string `json:"capabilities,omitempty"` + PublicDomain string `json:"public-domain,omitempty"` + Account string `json:"account,omitempty"` + AccountHome string `json:"account-home,omitempty"` + AgentAccount string `json:"agent-account,omitempty"` + AgentHome string `json:"agent-account-home,omitempty"` + Adopted bool `json:"adopted,omitempty"` + Taken []string `json:"taken,omitempty"` + SealingKey string `json:"sealing-key,omitempty"` + ServingKey string `json:"serving-key,omitempty"` + Certificate string `json:"certificate,omitempty"` + OutwardLinks []string `json:"outward-links,omitempty"` + Networks []catalogue.AttachedNetwork `json:"networks,omitempty"` + NetworksSaid bool `json:"networks-said,omitempty"` + Contract int `json:"contract,omitempty"` + Tunnel *RecordTunnel `json:"tunnel,omitempty"` + HubAdopted bool `json:"hub-adopted,omitempty"` + Carried []RecordCarriedPeer `json:"carried,omitempty"` +} + +func (r NodeRecord) check() error { + if err := checkName("node", r.Node); err != nil { + return err + } + if !r.Place.Hub && (r.HubAdopted || len(r.Carried) > 0) { + return fmt.Errorf("delivery: the record of %s carries peers or an adopted tunnel, which only the hub's does", r.Node) + } + if r.HubAdopted && r.Tunnel == nil { + return fmt.Errorf("delivery: the record of %s says the hub adopted its tunnel, and names none", r.Node) + } + for _, c := range r.Capabilities { + if c == "" { + return fmt.Errorf("delivery: the record of %s names an empty capability", r.Node) + } + } + return nil +} + +// EncodeNodeRecord is r as JSON at Version; it refuses what DecodeNodeRecord refuses. +func EncodeNodeRecord(r NodeRecord) ([]byte, error) { + if err := r.check(); err != nil { + return nil, err + } + r.V = Version + return json.Marshal(r) +} + +// DecodeNodeRecord reads a node record; it refuses an unknown version (ErrUnknownVersion), a node that is not a name, +// carried peers or an adopted tunnel on a node that is not the hub, and an empty capability. +func DecodeNodeRecord(data []byte) (NodeRecord, error) { + var r NodeRecord + if err := decode(data, &r); err != nil { + return NodeRecord{}, err + } + if err := r.check(); err != nil { + return NodeRecord{}, err + } + return r, nil +} + +// SameNodeRecord says whether two records are the same value, so the agent writes only when they differ. +func SameNodeRecord(a, b NodeRecord) bool { + a.V, b.V = 0, 0 + return jsonEqual(a, b) +} diff --git a/go/delivery/noderecord_test.go b/go/delivery/noderecord_test.go new file mode 100644 index 0000000..e882f76 --- /dev/null +++ b/go/delivery/noderecord_test.go @@ -0,0 +1,79 @@ +package delivery + +import ( + "reflect" + "regexp" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// secretNamed is facts.secretKey's pattern: a field so named would read as a secret, and the runtime's credential-field +// check (ADR 0201 §13) refuses such a field in a state every node reads. +var secretNamed = regexp.MustCompile(`(?i)(secret|passw|token|api[-_]?key|private[-_]?key|credential|bearer|cookie|salt|signing)`) + +// Issue 1126 (L1 as narrowed by ADR 0549): a node's record keys, reads back, refuses what only the hub may carry, and +// names no field like a secret. +func TestNodeRecord(t *testing.T) { + key, err := NodeRecordKey("ace") + if err != nil || key != "record.ace" || NodeRecordsBucket != "node-engine_records" { + t.Fatalf("NodeRecordKey = %q, %v; bucket %q", key, err, NodeRecordsBucket) + } + if n, err := ParseNodeRecordKey(key); err != nil || n != "ace" { + t.Fatalf("ParseNodeRecordKey = %q, %v", n, err) + } + for _, bad := range []string{"placement.ace", "record.", "record.a.b"} { + if _, err := ParseNodeRecordKey(bad); err == nil { + t.Errorf("ParseNodeRecordKey(%q) took it", bad) + } + } + + hub := NodeRecord{Node: "novox", Place: NodePlace{Key: "k", Endpoint: "192.0.2.1:51820", Hub: true, Address: "10.0.0.1"}, + Capabilities: []string{"docker"}, PublicDomain: "example.org", Account: "op", AccountHome: "/home/op", + AgentAccount: "agent", AgentHome: "/var/lib/agent", SealingKey: "age1x", ServingKey: "pub", Certificate: "cert", + OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, + NetworksSaid: true, Contract: 3, Taken: []string{"x"}, Adopted: true, + Tunnel: &RecordTunnel{Interface: "wg0", PublicKey: "k", Peers: []RecordTunnelPeer{{PublicKey: "p"}}}, + HubAdopted: true, Carried: []RecordCarriedPeer{{PublicKey: "p", EnrolledAs: "phone"}}} + raw, err := EncodeNodeRecord(hub) + if err != nil { + t.Fatal(err) + } + back, err := DecodeNodeRecord(raw) + if err != nil || !SameNodeRecord(back, hub) || back.V != Version { + t.Fatalf("round trip: %+v, %v", back, err) + } + + for name, r := range map[string]NodeRecord{ + "carried off the hub": {Node: "ace", Carried: []RecordCarriedPeer{{PublicKey: "p"}}}, + "adopted off the hub": {Node: "ace", HubAdopted: true}, + "adopted with no tunnel": {Node: "novox", Place: NodePlace{Hub: true}, HubAdopted: true}, + "empty capability": {Node: "ace", Capabilities: []string{""}}, + "node that is not a name": {Node: "a.b"}, + } { + if _, err := EncodeNodeRecord(r); err == nil { + t.Errorf("%s: taken", name) + } + } + + // Every JSON field name of the record and the types it carries, walked: none reads as a secret. + var walk func(reflect.Type, string) + walk = func(ty reflect.Type, at string) { + for ty.Kind() == reflect.Pointer || ty.Kind() == reflect.Slice { + ty = ty.Elem() + } + if ty.Kind() != reflect.Struct { + return + } + for i := 0; i < ty.NumField(); i++ { + f := ty.Field(i) + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if secretNamed.MatchString(name) || secretNamed.MatchString(f.Name) { + t.Errorf("%s.%s (%q) is named like a secret", at, f.Name, name) + } + walk(f.Type, at+"."+f.Name) + } + } + walk(reflect.TypeOf(NodeRecord{}), "NodeRecord") +} diff --git a/go/delivery/placement.go b/go/delivery/placement.go new file mode 100644 index 0000000..6e3c5cd --- /dev/null +++ b/go/delivery/placement.go @@ -0,0 +1,371 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "path" + "sort" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Placement of modules on nodes lives in the catalogue (novox/hq ADR 0549): one file per node, `placement/.json` +// in the catalogue repository, changed only by a pull request. At each trunk merge of the catalogue the indexer reads +// the files and writes each node's key `placement.` in its state `placement`, only when it differs, and says +// `placement-changed` naming the node, the revision and the modules added, removed or changed (ADR 0549 §2). The node +// agents, the assigner, the grants part and mesh-config read it (§3); nothing in it is secret. +const ( + // PlacementState is the indexer's state of placements, and PlacementBucket its bucket. + PlacementState = "placement" + PlacementBucket = IndexerModule + "_" + PlacementState + // PlacementHistory is the revisions the placement bucket keeps per key. + PlacementHistory = 10 + // PlacementDir is the catalogue's directory of placement files, one `.json` per node. + PlacementDir = "placement" + // PlacementChangedEvent is what the indexer says when a node's placement key changed. + PlacementChangedEvent = "placement-changed" + + placementWord = "placement" +) + +// PlacementKey is the key, in PlacementBucket, of one node's placement: `placement.`. +func PlacementKey(node string) (string, error) { + if err := checkName("node", node); err != nil { + return "", err + } + return placementWord + "." + node, nil +} + +// ParsePlacementKey is the inverse of PlacementKey; it refuses any other key, naming it. +func ParsePlacementKey(key string) (node string, err error) { + node, ok := strings.CutPrefix(key, placementWord+".") + if !ok || checkName("node", node) != nil { + return "", fmt.Errorf("delivery: %q is not a placement key, placement.", key) + } + return node, nil +} + +// PlacementFile is the path, in the catalogue repository, of one node's placement file. +func PlacementFile(node string) string { return path.Join(PlacementDir, node+".json") } + +// PlacementFileNode is the node a catalogue path is the placement file of, and whether it is one: `placement/.json` +// directly under PlacementDir. +func PlacementFileNode(p string) (string, bool) { + dir, file := path.Split(p) + node, ok := strings.CutSuffix(file, ".json") + if path.Clean(dir) != PlacementDir || !ok || checkName("node", node) != nil { + return "", false + } + return node, true +} + +// Provider is the module on a node that answers a requirement or holds a seat: a pin's or a binding's provider. +type Provider struct { + Node string `json:"node"` + Module string `json:"module"` +} + +// Chosen is p as the composition names a chosen provider. +func (p Provider) Chosen() catalogue.Chosen { return catalogue.Chosen{Node: p.Node, Module: p.Module} } + +// PlacedModule is one module placed on the node: the host port each port it declares is published on (by the port the +// software listens on), and, by provision, the provider each of its requirements that keeps data is bound to. +type PlacedModule struct { + Module string `json:"module"` + Ports map[string]int `json:"ports,omitempty"` + Bindings map[string]Provider `json:"bindings,omitempty"` +} + +// Holding is one seat a module placed on the node holds (catalogue.Held without its node, which is the file's). +type Holding struct { + Claim string `json:"claim"` + Scope string `json:"scope,omitempty"` + Module string `json:"module"` + Site string `json:"site,omitempty"` + Kind string `json:"kind,omitempty"` +} + +// Placement is one node's placement: the file the catalogue keeps and, with Revision, the value of its key. Modules are +// in the order the file names them, which is the order the composition reads the node's modules in. Pins is, by +// provision, the provider chosen for the node where more than one could answer. +type Placement struct { + V int `json:"v"` + Node string `json:"node"` + Revision string `json:"revision,omitempty"` + Modules []PlacedModule `json:"modules"` + Pins map[string]Provider `json:"pins,omitempty"` + Holds []Holding `json:"holds,omitempty"` +} + +func (p Placement) check() error { + if err := checkName("node", p.Node); err != nil { + return err + } + seen := map[string]bool{} + for _, m := range p.Modules { + if err := checkName("module", m.Module); err != nil { + return err + } + if seen[m.Module] { + return fmt.Errorf("delivery: the placement of %s names %s twice", p.Node, m.Module) + } + seen[m.Module] = true + for port, at := range m.Ports { + if n, err := strconv.Atoi(port); err != nil || n <= 0 || n > 65535 { + return fmt.Errorf("delivery: %s's port %q on %s is not a port number", m.Module, port, p.Node) + } + if at <= 0 || at > 65535 { + return fmt.Errorf("delivery: %s's port %s on %s is published on %d, which is not a port number", m.Module, port, p.Node, at) + } + } + for provision, to := range m.Bindings { + if provision == "" || to.Node == "" || to.Module == "" { + return fmt.Errorf("delivery: %s on %s has a binding with no provision, node or module", m.Module, p.Node) + } + } + } + for provision, to := range p.Pins { + if provision == "" || to.Node == "" || to.Module == "" { + return fmt.Errorf("delivery: the placement of %s has a pin with no provision, node or module", p.Node) + } + } + for _, h := range p.Holds { + if h.Claim == "" || !seen[h.Module] { + return fmt.Errorf("delivery: the placement of %s has %s hold %q, which is not a module placed there or names no seat", p.Node, h.Module, h.Claim) + } + } + return nil +} + +// EncodePlacement is p as JSON at Version; it refuses what DecodePlacement refuses. +func EncodePlacement(p Placement) ([]byte, error) { + if err := p.check(); err != nil { + return nil, err + } + p.V = Version + return json.Marshal(p) +} + +// DecodePlacement reads a placement file or key value; it refuses an unknown version (ErrUnknownVersion), a node or +// module that is not a name, a module named twice, a port that is not a number, a binding or pin naming no provider, +// and a holding by a module not placed on the node. +func DecodePlacement(data []byte) (Placement, error) { + var p Placement + if err := decode(data, &p); err != nil { + return Placement{}, err + } + if err := p.check(); err != nil { + return Placement{}, err + } + return p, nil +} + +// SamePlacement says whether two placements are the same value, so the indexer writes only when they differ. The +// revision is not compared: a merge that changes no placement writes nothing (ADR 0549 §2). +func SamePlacement(a, b Placement) bool { + a.Revision, b.Revision = "", "" + ja, errA := EncodePlacement(a) + jb, errB := EncodePlacement(b) + return errA == nil && errB == nil && string(ja) == string(jb) +} + +// PlacementChange is what `placement-changed` says: the node, the catalogue revision, and its modules added, removed +// or changed against the placement before. +type PlacementChange struct { + Node string `json:"node"` + Revision string `json:"revision"` + Added []string `json:"added,omitempty"` + Removed []string `json:"removed,omitempty"` + Changed []string `json:"changed,omitempty"` +} + +// ChangeOf is what changed from before to after on one node; before is the zero Placement for a node new to placement. +// A change of the node's pins or holdings counts as a change of every module placed on it both times, since the +// composition of each may read them. +func ChangeOf(before, after Placement) PlacementChange { + c := PlacementChange{Node: after.Node, Revision: after.Revision} + if c.Node == "" { + c.Node = before.Node + } + was := map[string]PlacedModule{} + for _, m := range before.Modules { + was[m.Module] = m + } + nodeWide := !jsonEqual(before.Pins, after.Pins) || !jsonEqual(before.Holds, after.Holds) + now := map[string]bool{} + for _, m := range after.Modules { + now[m.Module] = true + old, ok := was[m.Module] + switch { + case !ok: + c.Added = append(c.Added, m.Module) + case nodeWide || !jsonEqual(old, m): + c.Changed = append(c.Changed, m.Module) + } + } + for _, m := range before.Modules { + if !now[m.Module] { + c.Removed = append(c.Removed, m.Module) + } + } + sort.Strings(c.Added) + sort.Strings(c.Removed) + sort.Strings(c.Changed) + return c +} + +func jsonEqual(a, b any) bool { + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + return string(ja) == string(jb) +} + +// Assigned is the modules placed on the node, in the file's order (compose.Node's Assigned). +func (p Placement) Assigned() []string { + out := make([]string, 0, len(p.Modules)) + for _, m := range p.Modules { + out = append(out, m.Module) + } + return out +} + +// ChosenPins is the node's pins as the composition reads them (compose.Node's Pins); nil when it has none. +func (p Placement) ChosenPins() map[string]catalogue.Chosen { + if len(p.Pins) == 0 { + return nil + } + out := make(map[string]catalogue.Chosen, len(p.Pins)) + for provision, to := range p.Pins { + out[provision] = to.Chosen() + } + return out +} + +// Bound is, by consumer module and provision, where each binding sends it (compose.Node's Bound); nil when none. +func (p Placement) Bound() map[string]map[string]catalogue.Chosen { + var out map[string]map[string]catalogue.Chosen + for _, m := range p.Modules { + if len(m.Bindings) == 0 { + continue + } + if out == nil { + out = map[string]map[string]catalogue.Chosen{} + } + out[m.Module] = make(map[string]catalogue.Chosen, len(m.Bindings)) + for provision, to := range m.Bindings { + out[m.Module][provision] = to.Chosen() + } + } + return out +} + +// HostPorts is, by module and the port it declares, the host port it is published on (compose.Node's Ports); nil when +// none. The check has refused a port that is not a number, so none is dropped here. +func (p Placement) HostPorts() map[string]map[int]int { + var out map[string]map[int]int + for _, m := range p.Modules { + if len(m.Ports) == 0 { + continue + } + if out == nil { + out = map[string]map[int]int{} + } + out[m.Module] = make(map[int]int, len(m.Ports)) + for port, at := range m.Ports { + n, _ := strconv.Atoi(port) + out[m.Module][n] = at + } + } + return out +} + +// Held is the node's holdings as the composition reads them (compose.Input's Holdings rows for this node). +func (p Placement) Held() []catalogue.Held { + out := make([]catalogue.Held, 0, len(p.Holds)) + for _, h := range p.Holds { + out = append(out, catalogue.Held{Claim: h.Claim, Scope: h.Scope, Node: p.Node, Module: h.Module, Site: h.Site, Kind: h.Kind}) + } + return out +} + +// CheckPlacements is the catalogue's own check of every placement file (ADR 0549 §1), over the placements by node, the +// modules the catalogue holds and the nodes enrolled. It returns every refusal, in node order, never only the first: a +// placement naming a node not enrolled or a module the catalogue does not hold; two modules of one node on the same host +// port; a pin, binding or holding naming a provider that is not placed where it says; a seat of scope mesh held twice. +// A host port a manifest declares without a number and a requirement left unbound are the composition's to find (its +// ForeseenPorts and Unbound), which the catalogue's composition check reads. +func CheckPlacements(placements map[string]Placement, held func(module string) bool, enrolled func(node string) bool) []string { + var refusals []string + nodes := make([]string, 0, len(placements)) + for n := range placements { + nodes = append(nodes, n) + } + sort.Strings(nodes) + placed := func(p Provider) bool { + other, ok := placements[p.Node] + if !ok { + return false + } + for _, m := range other.Modules { + if m.Module == p.Module { + return true + } + } + return false + } + meshSeats := map[string]string{} + for _, node := range nodes { + p := placements[node] + if p.Node != node { + refusals = append(refusals, fmt.Sprintf("the placement file of %s names the node %q", node, p.Node)) + } + if !enrolled(node) { + refusals = append(refusals, fmt.Sprintf("%s is not an enrolled node", node)) + } + hostPorts := map[int]string{} + for _, m := range p.Modules { + if !held(m.Module) { + refusals = append(refusals, fmt.Sprintf("%s: the catalogue holds no module %s", node, m.Module)) + } + for _, port := range sortedKeys(m.Ports) { + at := m.Ports[port] + if other, clash := hostPorts[at]; clash && other != m.Module { + refusals = append(refusals, fmt.Sprintf("%s: %s and %s are both published on host port %d", node, other, m.Module, at)) + } + hostPorts[at] = m.Module + } + for _, provision := range sortedKeys(m.Bindings) { + if to := m.Bindings[provision]; !placed(to) { + refusals = append(refusals, fmt.Sprintf("%s: %s's %s is bound to %s/%s, which is not placed there", node, m.Module, provision, to.Node, to.Module)) + } + } + } + for _, provision := range sortedKeys(p.Pins) { + if to := p.Pins[provision]; !placed(to) { + refusals = append(refusals, fmt.Sprintf("%s: the pin of %s names %s/%s, which is not placed there", node, provision, to.Node, to.Module)) + } + } + for _, h := range p.Holds { + if h.Scope != "mesh" { + continue + } + seat := h.Claim + "/" + h.Kind + if other, twice := meshSeats[seat]; twice { + refusals = append(refusals, fmt.Sprintf("the seat %s, held once for the mesh, is held on %s and %s", h.Claim, other, node)) + } + meshSeats[seat] = node + } + } + return refusals +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go new file mode 100644 index 0000000..8a97ac3 --- /dev/null +++ b/go/delivery/placement_test.go @@ -0,0 +1,152 @@ +package delivery + +import ( + "errors" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// ADR 0549 §1, §2: the placement key and file, the value's refusals, its reading as the composition reads a node, what +// a change says, and the catalogue's check of every file. +func TestPlacement(t *testing.T) { + t.Run("key and file", func(t *testing.T) { + key, err := PlacementKey("g14") + if err != nil || key != "placement.g14" { + t.Fatalf("PlacementKey = %q, %v", key, err) + } + if n, err := ParsePlacementKey(key); err != nil || n != "g14" { + t.Fatalf("ParsePlacementKey = %q, %v", n, err) + } + for _, bad := range []string{"want.g14.x", "placement.", "placement.a.b"} { + if _, err := ParsePlacementKey(bad); err == nil { + t.Errorf("ParsePlacementKey(%q) took it", bad) + } + } + if _, err := PlacementKey("a.b"); err == nil { + t.Error("PlacementKey took a dotted node") + } + if PlacementBucket != "mesh-delivery-indexer_placement" || PlacementFile("ace") != "placement/ace.json" { + t.Fatalf("bucket %q, file %q", PlacementBucket, PlacementFile("ace")) + } + for p, want := range map[string]string{"placement/ace.json": "ace", "./placement/ace.json": "ace", + "placement/x/ace.json": "", "modules/ace.json": "", "placement/ace.yaml": ""} { + if n, ok := PlacementFileNode(p); n != want || ok != (want != "") { + t.Errorf("PlacementFileNode(%q) = %q, %v", p, n, ok) + } + } + }) + + g14 := Placement{Node: "g14", Revision: "abc", + Modules: []PlacedModule{ + {Module: "zsh"}, + {Module: "app", Ports: map[string]int{"8080": 18080}, Bindings: map[string]Provider{"postgres": {Node: "novox", Module: "postgres"}}}, + }, + Pins: map[string]Provider{"shell": {Node: "g14", Module: "zsh"}}, + Holds: []Holding{{Claim: "node-shell", Scope: "node", Module: "zsh"}}, + } + novox := Placement{Node: "novox", Modules: []PlacedModule{{Module: "postgres", Ports: map[string]int{"5432": 5432}}}, + Holds: []Holding{{Claim: "mesh-store", Scope: "mesh", Module: "postgres"}}} + + t.Run("value", func(t *testing.T) { + raw, err := EncodePlacement(g14) + if err != nil { + t.Fatal(err) + } + back, err := DecodePlacement(raw) + if err != nil || back.V != Version || !SamePlacement(back, g14) { + t.Fatalf("round trip: %+v, %v", back, err) + } + moved := g14 + moved.Revision = "def" + if !SamePlacement(g14, moved) { + t.Error("a new revision alone counts as a change") + } + if _, err := DecodePlacement([]byte(`{"v":9,"node":"g14","modules":[]}`)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("version 9: %v", err) + } + for name, bad := range map[string]string{ + "module twice": `{"v":1,"node":"g14","modules":[{"module":"a"},{"module":"a"}]}`, + "port not number": `{"v":1,"node":"g14","modules":[{"module":"a","ports":{"http":80}}]}`, + "host port zero": `{"v":1,"node":"g14","modules":[{"module":"a","ports":{"80":0}}]}`, + "binding no node": `{"v":1,"node":"g14","modules":[{"module":"a","bindings":{"pg":{"module":"pg"}}}]}`, + "pin no module": `{"v":1,"node":"g14","modules":[],"pins":{"pg":{"node":"x"}}}`, + "hold not placed": `{"v":1,"node":"g14","modules":[],"holds":[{"claim":"s","module":"a"}]}`, + "node not a name": `{"v":1,"node":"g.14","modules":[]}`, + "value not shaped": `{"v":1,"node":"g14","modules":{}}`, + } { + if _, err := DecodePlacement([]byte(bad)); err == nil { + t.Errorf("%s: taken", name) + } + } + }) + + t.Run("as the composition reads it", func(t *testing.T) { + if got := g14.Assigned(); !reflect.DeepEqual(got, []string{"zsh", "app"}) { + t.Errorf("Assigned = %v, want the file's order", got) + } + if got := g14.ChosenPins(); !reflect.DeepEqual(got, map[string]catalogue.Chosen{"shell": {Node: "g14", Module: "zsh"}}) { + t.Errorf("ChosenPins = %v", got) + } + if got := g14.Bound(); !reflect.DeepEqual(got, map[string]map[string]catalogue.Chosen{"app": {"postgres": {Node: "novox", Module: "postgres"}}}) { + t.Errorf("Bound = %v", got) + } + if got := g14.HostPorts(); !reflect.DeepEqual(got, map[string]map[int]int{"app": {8080: 18080}}) { + t.Errorf("HostPorts = %v", got) + } + if got := g14.Held(); !reflect.DeepEqual(got, []catalogue.Held{{Claim: "node-shell", Scope: "node", Node: "g14", Module: "zsh"}}) { + t.Errorf("Held = %v", got) + } + empty := Placement{Node: "x"} + if empty.ChosenPins() != nil || empty.Bound() != nil || empty.HostPorts() != nil || len(empty.Held()) != 0 { + t.Error("an empty placement reads as something") + } + }) + + t.Run("what a change says", func(t *testing.T) { + after := g14 + after.Revision = "def" + after.Modules = []PlacedModule{{Module: "app", Ports: map[string]int{"8080": 18081}}, {Module: "new"}} + got := ChangeOf(g14, after) + want := PlacementChange{Node: "g14", Revision: "def", Added: []string{"new"}, Removed: []string{"zsh"}, Changed: []string{"app"}} + if !reflect.DeepEqual(got, want) { + t.Errorf("ChangeOf = %+v, want %+v", got, want) + } + pinned := g14 + pinned.Pins = nil + if got := ChangeOf(g14, pinned); !reflect.DeepEqual(got.Changed, []string{"app", "zsh"}) { + t.Errorf("a pin change changes every module on the node: %+v", got) + } + if got := ChangeOf(Placement{}, novox); !reflect.DeepEqual(got.Added, []string{"postgres"}) || got.Node != "novox" { + t.Errorf("a new node: %+v", got) + } + }) + + t.Run("the catalogue's check", func(t *testing.T) { + held := func(m string) bool { return m != "ghost" } + enrolled := func(n string) bool { return n != "stranger" } + if r := CheckPlacements(map[string]Placement{"g14": g14, "novox": novox}, held, enrolled); len(r) != 0 { + t.Fatalf("a sound placement is refused: %v", r) + } + clash := g14 + clash.Modules = append([]PlacedModule{{Module: "ghost", Ports: map[string]int{"80": 18080}}}, g14.Modules...) + clash.Modules[2] = PlacedModule{Module: "app", Ports: map[string]int{"8080": 18080}, Bindings: map[string]Provider{"postgres": {Node: "ace", Module: "postgres"}}} + clash.Pins = map[string]Provider{"shell": {Node: "g14", Module: "bash"}} + twice := novox + twice.Node = "stranger" + r := CheckPlacements(map[string]Placement{"g14": clash, "novox": novox, "stranger": twice}, held, enrolled) + for _, want := range []string{"holds no module ghost", "ghost and app are both published on host port 18080", + "bound to ace/postgres", "pin of shell names g14/bash", "stranger is not an enrolled node", + "mesh-store, held once for the mesh, is held on novox and stranger"} { + found := false + for _, line := range r { + found = found || strings.Contains(line, want) + } + if !found { + t.Errorf("no refusal says %q; got %v", want, r) + } + } + }) +} From e9820724430bc4a64d206797ee33783679318972 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:06:04 +0100 Subject: [PATCH 02/42] =?UTF-8?q?frombus=20fills=20the=20composition's=20i?= =?UTF-8?q?nput=20from=20placement,=20node=20records,=20the=20indexer's=20?= =?UTF-8?q?manifests=20and=20mesh-config's=20slice,=20so=20one=20composer?= =?UTF-8?q?=20serves=20the=20node=20agent,=20the=20check=20and=20the=20gra?= =?UTF-8?q?nts=20part=20(issue=201126,=20ADR=200547=20=C2=A72,=20ADR=20054?= =?UTF-8?q?9=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/frombus/frombus.go | 255 +++++++++++++++++++++++++++++ go/compose/frombus/frombus_test.go | 181 ++++++++++++++++++++ go/delivery/delivery_test.go | 3 +- go/delivery/placement.go | 29 ++++ go/delivery/placement_test.go | 17 ++ 5 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 go/compose/frombus/frombus.go create mode 100644 go/compose/frombus/frombus_test.go diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go new file mode 100644 index 0000000..2e96c5b --- /dev/null +++ b/go/compose/frombus/frombus.go @@ -0,0 +1,255 @@ +// Package frombus fills mesh-sdk's composition input from what is on the bus, so a node agent, the composition check +// and the grants part compose with one composer over the same inputs (novox/hq ADR 0547 §2, ADR 0549 §3; issue 1126). +// +// **One composer, three callers.** compose.Compositions is the composer; Input is how each caller fills its +// compose.Input from the bus: the node agent for its own node (ADR 0547 §2), the assigner's composition check for every +// node (ADR 0539 §6's check, ADR 0547 §4), and the grants part (ADR 0547 §5). Each reads the bus with its own credential +// through a Reader, so this package holds no bus client, no store and no timer: what it is given is all it reads. +// +// **What it reads, and from whom.** +// - which modules run on each node, with their pins, holdings, host ports, bindings and settings placement: the +// indexer's state `placement`, one key per node (ADR 0549 §2); +// - what each node knows of itself (its place on the private network, capabilities, accounts, keys, networks, +// tunnel, and on the hub the adopted tunnel and its carried peers): the node-engine's state `records`; +// - every module's manifest: the indexer's `modules.` and its manifests by digest, or the digest the +// composing node's want names where it runs another build (Scope.Kept); +// - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the +// caller composes every node (ADR 0547 §2, the prepare step's L4). +// +// **Not yet read** (each a later subtask of issue 1126): licences, own secrets, issued credentials and core boxes, which +// the vault answers sealed to the node (ADR 0550 §3, §5). Without them the composition stands in for each and names it +// foreseen, as compose.Reading does for a value the Input does not hold. No node is joining before ADR 0547's third +// milestone (ADR 0550 §6), so Joining is always empty. +package frombus + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// Reader is the bus as a caller reads it with its own credential. Each method reads one part's state; an error is a +// read that failed, never an absent key (an absent key is an empty answer). +type Reader interface { + // Placements is every node's placement, by node (delivery.PlacementBucket). + Placements(ctx context.Context) (map[string]delivery.Placement, error) + // Records is every node's record, by node (delivery.NodeRecordsBucket). + Records(ctx context.Context) (map[string]delivery.NodeRecord, error) + // Modules is every module the indexer holds, by module (delivery.ModulesBucket). + Modules(ctx context.Context) (map[string]delivery.Module, error) + // Manifest is the raw manifest of a digest (delivery.ManifestsBucket). + Manifest(ctx context.Context, digest string) ([]byte, error) + // Values is one module's config values on one node as mesh-config resolved them (mesh-config's state `resolved`, + // keys `..`), decoded; nil when it has none. + Values(ctx context.Context, node, module string) (map[string]any, error) +} + +// Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the +// internal names' suffix as genesis was told them, and the mesh's authority certificate (public). Empty fields take +// compose's defaults. +type Mesh struct { + OverlayCIDR string + InternalSuffix string + Authority string + // Predefined is the list of predefined keys mesh-config published, which a manifest's values falling back to a + // predefined key are judged against; nil reads each manifest as the indexer does (catalogue.ParseManifestForInputs). + Predefined catalogue.PredefinedList +} + +// Scope says who composes. Node is the node composing; only its config values are read unless Every is set, as the +// composition check sets it to compose every node. Kept is, by module, the manifest digest the composing node's wants +// name where it differs from the indexer's trunk: that node composes the module as it runs it. +type Scope struct { + Node string + Every bool + Kept map[string]string +} + +// Input is the composition input for s, read through r. +func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, error) { + placements, err := r.Placements(ctx) + if err != nil { + return compose.Input{}, fmt.Errorf("frombus: placement: %w", err) + } + records, err := r.Records(ctx) + if err != nil { + return compose.Input{}, fmt.Errorf("frombus: node records: %w", err) + } + modules, err := r.Modules(ctx) + if err != nil { + return compose.Input{}, fmt.Errorf("frombus: the indexer's modules: %w", err) + } + if _, ok := placements[s.Node]; !ok && !s.Every { + return compose.Input{}, fmt.Errorf("frombus: %s has no placement in the catalogue", s.Node) + } + + in := compose.Input{OverlayCIDR: mesh.OverlayCIDR, InternalSuffix: mesh.InternalSuffix, Authority: mesh.Authority, + Catalogue: map[string]catalogue.Manifest{}} + read := func(module, digest string) (catalogue.Manifest, error) { + raw, err := r.Manifest(ctx, digest) + if err != nil { + return catalogue.Manifest{}, fmt.Errorf("frombus: the manifest of %s (digest %s): %w", module, digest, err) + } + if mesh.Predefined != nil { + return catalogue.ParseManifestWith(raw, mesh.Predefined) + } + return catalogue.ParseManifestForInputs(raw) + } + for _, name := range sortedKeys(modules) { + m := modules[name] + if m.Manifest == "" { + continue // a registered module the indexer has not read: nothing to compose it from + } + man, err := read(name, m.Manifest) + if err != nil { + return compose.Input{}, err + } + in.Catalogue[name] = man + } + + // The module the mesh provides itself (the private network) has no repository the indexer reads: its manifest comes + // with mesh-sdk and always wins, as the controller's providedModules and the grants part have it. + provided, err := json.Marshal(overlay.Manifest()) + if err != nil { + return compose.Input{}, err + } + var private catalogue.Manifest + if err := json.Unmarshal(provided, &private); err != nil { + return compose.Input{}, fmt.Errorf("frombus: the private network's manifest mesh-sdk provides: %w", err) + } + in.Catalogue[overlay.Name] = private + + var missing []string + for _, node := range nodesOf(placements, records) { + rec, hasRecord := records[node] + p := placements[node] + if !hasRecord { + if len(p.Modules) > 0 { + missing = append(missing, node) + } + continue + } + n := nodeOf(rec, p) + withValues := s.Every || node == s.Node + for _, m := range p.Modules { + var values map[string]any + if withValues { + if values, err = r.Values(ctx, node, m.Module); err != nil { + return compose.Input{}, fmt.Errorf("frombus: %s's config values on %s: %w", m.Module, node, err) + } + } + if layers := m.Layers(node, values); layers != nil { + if n.Settings == nil { + n.Settings = map[string][]catalogue.Layer{} + } + n.Settings[m.Module] = layers + } + } + if node == s.Node { + for _, module := range sortedKeys(s.Kept) { + man, err := read(module, s.Kept[module]) + if err != nil { + return compose.Input{}, err + } + if n.Kept == nil { + n.Kept = map[string]catalogue.Manifest{} + } + n.Kept[module] = man + } + } + in.Nodes = append(in.Nodes, n) + in.Holdings = append(in.Holdings, p.Held()...) + if rec.Tunnel != nil { + if in.Tunnels == nil { + in.Tunnels = map[string]compose.FoundTunnel{} + } + in.Tunnels[node] = compose.FoundTunnel{Tunnel: tunnelOf(*rec.Tunnel), NodeAdopted: rec.Adopted} + if rec.Place.Hub { + in.Adopted = &compose.AdoptedTunnel{Tunnel: tunnelOf(*rec.Tunnel), Hub: node, Adopted: rec.HubAdopted} + } + } + if rec.Place.Hub { + for _, c := range rec.Carried { + in.CarriedPeers = append(in.CarriedPeers, compose.CarriedPeer{PublicKey: c.PublicKey, Address: c.Address, + EnrolledAs: c.EnrolledAs, Named: c.Named}) + } + } + } + if len(missing) > 0 { + return compose.Input{}, fmt.Errorf("frombus: modules are placed on %s, whose agent has written no record yet", + strings.Join(missing, ", ")) + } + if !s.Every { + if _, ok := records[s.Node]; !ok { + return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) + } + } + return in, nil +} + +// Compose is the node's compositions over the bus (ADR 0547 §2): Input for s, then compose.Compositions in the Reading +// mode at the fingerprints fps gives each module, as the controller composes them. +func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]string) (map[string]delivery.Composed, []string, error) { + in, err := Input(ctx, r, mesh, s) + if err != nil { + return nil, nil, err + } + return compose.Compositions(in, s.Node, fps, compose.Reading) +} + +// nodeOf is the composition's node from its record and its placement. +func nodeOf(rec delivery.NodeRecord, p delivery.Placement) compose.Node { + n := compose.Node{ + Name: rec.Node, + Place: compose.Place{Key: rec.Place.Key, Endpoint: rec.Place.Endpoint, Site: rec.Place.Site, Hub: rec.Place.Hub, Address: rec.Place.Address}, + Assigned: p.Assigned(), PublicDomain: rec.PublicDomain, + Account: rec.Account, AccountHome: rec.AccountHome, AgentAccount: rec.AgentAccount, AgentAccountHome: rec.AgentHome, + Adopted: rec.Adopted, Pins: p.ChosenPins(), Bound: p.Bound(), Ports: p.HostPorts(), SealingKey: rec.SealingKey, + Taken: rec.Taken, OutwardLinks: rec.OutwardLinks, Networks: rec.Networks, NetworksSaid: rec.NetworksSaid, + ServingKey: rec.ServingKey, Certificate: rec.Certificate, Contract: rec.Contract, Stated: rec.Contract > 0, + } + if len(rec.Capabilities) > 0 { + n.Capabilities = make(map[string]bool, len(rec.Capabilities)) + for _, c := range rec.Capabilities { + n.Capabilities[c] = true + } + } + return n +} + +func tunnelOf(t delivery.RecordTunnel) compose.Tunnel { + out := compose.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU, + Address: t.Address, Range: t.Range, PublicKey: t.PublicKey} + for _, p := range t.Peers { + out.Peers = append(out.Peers, compose.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address}) + } + return out +} + +// nodesOf is every node named by a placement or a record, by name: the order the controller reads them in. +func nodesOf(placements map[string]delivery.Placement, records map[string]delivery.NodeRecord) []string { + seen := map[string]bool{} + for n := range placements { + seen[n] = true + } + for n := range records { + seen[n] = true + } + return sortedKeys(seen) +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go new file mode 100644 index 0000000..93d8f1e --- /dev/null +++ b/go/compose/frombus/frombus_test.go @@ -0,0 +1,181 @@ +package frombus + +import ( + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// bus is a Reader over values held in memory, counting whose config values were read. +type bus struct { + placements map[string]delivery.Placement + records map[string]delivery.NodeRecord + modules map[string]delivery.Module + manifests map[string][]byte + values map[string]map[string]any // by node.module + valuesRead []string +} + +func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) { + return b.placements, nil +} +func (b *bus) Records(context.Context) (map[string]delivery.NodeRecord, error) { return b.records, nil } +func (b *bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.modules, nil } +func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { + raw, ok := b.manifests[digest] + if !ok { + return nil, errors.New("no such manifest") + } + return raw, nil +} +func (b *bus) Values(_ context.Context, node, module string) (map[string]any, error) { + b.valuesRead = append(b.valuesRead, node+"."+module) + return b.values[node+"."+module], nil +} + +func publicKey(t *testing.T) string { + t.Helper() + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()) +} + +// Issue 1126 (S3, S4, S5): the bus's placement, node records, manifests and config values fill the same compose.Input the +// controller's store fills for compose's own two-machine mesh (compose's aMesh: anchor the hub, laptop), and that Input +// composes; a node reads only its own config values; what cannot be composed is refused, naming it. +func TestInput(t *testing.T) { + ctx := context.Background() + xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") + if err != nil { + t.Fatal(err) + } + digest := delivery.ManifestDigest(xterm) + keys := map[string]string{"anchor": publicKey(t), "laptop": publicKey(t)} + caps := []string{"container-runtime", "systemd", "wireguard"} + tunnel := &delivery.RecordTunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", + Peers: []delivery.RecordTunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}}} + mk := func() *bus { + return &bus{ + placements: map[string]delivery.Placement{ + "anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: overlay.Name}}, + Holds: []delivery.Holding{{Claim: "mesh-terminal", Scope: "mesh", Module: "xterm"}}}, + "laptop": {Node: "laptop", Modules: []delivery.PlacedModule{{Module: overlay.Name}, + {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}}}, + }, + records: map[string]delivery.NodeRecord{ + "anchor": {Node: "anchor", Place: delivery.NodePlace{Key: keys["anchor"], Endpoint: "anchor.example:51820", + Site: "here", Hub: true, Address: "10.77.0.1"}, Capabilities: caps, Tunnel: tunnel, Contract: 2, + Carried: []delivery.RecordCarriedPeer{{PublicKey: "peer", Address: "10.77.0.9", EnrolledAs: "phone"}}}, + "laptop": {Node: "laptop", Place: delivery.NodePlace{Key: keys["laptop"], Endpoint: "laptop.example:51820", + Site: "here", Address: "10.77.0.2"}, Capabilities: caps}, + }, + modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "unread": {}}, + manifests: map[string][]byte{digest: xterm}, + values: map[string]map[string]any{"laptop.xterm": {"font": "mono"}, "anchor.xterm": {"font": "serif"}}, + } + } + + // What the controller's store gives for the same mesh, as compose's aMesh and SettingsFor build it. + var private catalogue.Manifest + raw, _ := json.Marshal(overlay.Manifest()) + if err := json.Unmarshal(raw, &private); err != nil { + t.Fatal(err) + } + parsed, err := catalogue.ParseManifestForInputs(xterm) + if err != nil { + t.Fatal(err) + } + capsSet := map[string]bool{"container-runtime": true, "systemd": true, "wireguard": true} + tun := compose.Tunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", + Peers: []compose.TunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}}} + want := compose.Input{ + Catalogue: map[string]catalogue.Manifest{overlay.Name: private, "xterm": parsed}, + Nodes: []compose.Node{ + {Name: "anchor", Place: compose.Place{Key: keys["anchor"], Endpoint: "anchor.example:51820", Site: "here", Hub: true, + Address: "10.77.0.1"}, Assigned: []string{overlay.Name}, Capabilities: capsSet, Contract: 2, Stated: true}, + {Name: "laptop", Place: compose.Place{Key: keys["laptop"], Endpoint: "laptop.example:51820", Site: "here", + Address: "10.77.0.2"}, Assigned: []string{overlay.Name, "xterm"}, Capabilities: capsSet, + Ports: map[string]map[int]int{"xterm": {80: 8080}}, + Settings: map[string][]catalogue.Layer{"xterm": {{From: "laptop", Values: map[string]any{"font": "mono"}}}}}, + }, + Holdings: []catalogue.Held{{Claim: "mesh-terminal", Scope: "mesh", Node: "anchor", Module: "xterm"}}, + Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, + Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor"}, + CarriedPeers: []compose.CarriedPeer{{PublicKey: "peer", Address: "10.77.0.9", EnrolledAs: "phone"}}, + } + + b := mk() + got, err := Input(ctx, b, Mesh{}, Scope{Node: "laptop"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + gj, _ := json.MarshalIndent(got, "", " ") + wj, _ := json.MarshalIndent(want, "", " ") + t.Fatalf("Input from the bus differs from the store's:\n got %s\nwant %s", gj, wj) + } + if !reflect.DeepEqual(b.valuesRead, []string{"laptop." + overlay.Name, "laptop.xterm"}) { + t.Errorf("laptop read config values %v; a node reads its own slice alone", b.valuesRead) + } + + // The same composer over both: the compositions from the bus are the store's, byte for byte. + fps := map[string]string{overlay.Name: "src2:" + strings.Repeat("a", 64), "xterm": "src2:" + strings.Repeat("b", 64)} + fromBus, _, err := Compose(ctx, mk(), Mesh{}, Scope{Node: "laptop"}, fps) + if err != nil { + t.Fatal(err) + } + fromStore, _, err := compose.Compositions(want, "laptop", fps, compose.Reading) + if err != nil { + t.Fatal(err) + } + if len(fromBus) == 0 || !reflect.DeepEqual(fromBus, fromStore) { + t.Fatalf("compositions differ or are empty: %d from the bus, %d from the store", len(fromBus), len(fromStore)) + } + + b = mk() + if _, err := Input(ctx, b, Mesh{}, Scope{Every: true}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(b.valuesRead, []string{"anchor." + overlay.Name, "laptop." + overlay.Name, "laptop.xterm"}) { + // Every node's placed modules, and nothing placed nowhere: anchor places no xterm. + t.Errorf("the check read %v", b.valuesRead) + } + + kept := mk() + kept.manifests["old"] = xterm + in, err := Input(ctx, kept, Mesh{}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "old"}}) + if err != nil || !reflect.DeepEqual(in.Nodes[1].Kept, map[string]catalogue.Manifest{"xterm": parsed}) || in.Nodes[0].Kept != nil { + t.Errorf("the want's manifest is kept on the composing node alone: %v, %v", in.Nodes, err) + } + + for name, c := range map[string]struct { + change func(*bus) + scope Scope + says string + }{ + "no record": {func(b *bus) { delete(b.records, "laptop") }, Scope{Node: "anchor"}, "laptop, whose agent has written no record"}, + "no placement": {func(b *bus) {}, Scope{Node: "stranger"}, "stranger has no placement"}, + "a manifest unread": {func(b *bus) { delete(b.manifests, digest) }, Scope{Node: "laptop"}, "the manifest of xterm"}, + "a kept one unread": {func(b *bus) {}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "gone"}}, "digest gone"}, + } { + b := mk() + c.change(b) + if _, err := Input(ctx, b, Mesh{}, c.scope); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: %v, want it to say %q", name, err, c.says) + } + } +} diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index 99cfeba..f455eb3 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -16,6 +16,7 @@ import ( func TestHaveKey(t *testing.T) { t.Run("presence", testPresenceKey) t.Run("pulse", testPulseKey) + t.Run("pulse ttl", testPulseTTL) if HaveBucket != "node-engine_have" { t.Fatalf("HaveBucket = %q", HaveBucket) } @@ -368,7 +369,7 @@ func TestPhaseHealthRefused(t *testing.T) { } // 1131/AC7: the pulse's time to live is 120 seconds minus the move's age; at the bound the move is stalled. -func TestPulseTTL(t *testing.T) { +func testPulseTTL(t *testing.T) { for _, c := range []struct { age time.Duration ttl time.Duration diff --git a/go/delivery/placement.go b/go/delivery/placement.go index 6e3c5cd..3659b3b 100644 --- a/go/delivery/placement.go +++ b/go/delivery/placement.go @@ -72,10 +72,21 @@ func (p Provider) Chosen() catalogue.Chosen { return catalogue.Chosen{Node: p.No // PlacedModule is one module placed on the node: the host port each port it declares is published on (by the port the // software listens on), and, by provision, the provider each of its requirements that keeps data is bound to. +// +// MeshWide and Layer are where the module's things are, as a settings layer's placement holds them (catalogue.Layer's +// Placement: `places`, `accesses`, `networks`, `endpoints`, `expose`, `reach`, `ports`; ADR 0348 §11): the whole mesh's +// layer and this node's, which the controller keeps in its table `placement` (ADR 0549 §6 names it) and composes as two +// layers, the mesh's first. Each node's file repeats the mesh-wide layer of the modules placed on it, so a node reads +// its own key alone. HasLayer says the node has a layer of its own even when it holds no placement, since a layer +// that is there and empty still carries the module's config values (Layers). type PlacedModule struct { Module string `json:"module"` Ports map[string]int `json:"ports,omitempty"` Bindings map[string]Provider `json:"bindings,omitempty"` + MeshWide map[string]any `json:"mesh-wide,omitempty"` + HasMesh bool `json:"has-mesh-wide,omitempty"` + Layer map[string]any `json:"layer,omitempty"` + HasLayer bool `json:"has-layer,omitempty"` } // Holding is one seat a module placed on the node holds (catalogue.Held without its node, which is the file's). @@ -281,6 +292,24 @@ func (p Placement) HostPorts() map[string]map[int]int { return out } +// Layers is one module's settings layers on the node, as the composition reads them (compose.Node's Settings): the +// whole mesh's layer when it has one, then the node's, which carries the module's config values as mesh-config resolved +// them for this node (values, nil when none) beside its placement; a module with no layer and no value has none. This is +// the controller's SettingsFor for a module that takes its config values from mesh-config (issue 664). +func (m PlacedModule) Layers(node string, values map[string]any) []catalogue.Layer { + var out []catalogue.Layer + if m.HasMesh || len(m.MeshWide) > 0 { + out = append(out, catalogue.Layer{From: catalogue.MeshWideLayer, Values: map[string]any{}, Placement: m.MeshWide}) + } + if m.HasLayer || len(m.Layer) > 0 || len(values) > 0 { + if values == nil { + values = map[string]any{} + } + out = append(out, catalogue.Layer{From: node, Values: values, Placement: m.Layer}) + } + return out +} + // Held is the node's holdings as the composition reads them (compose.Input's Holdings rows for this node). func (p Placement) Held() []catalogue.Held { out := make([]catalogue.Held, 0, len(p.Holds)) diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index 8a97ac3..40c6eb3 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -99,6 +99,23 @@ func TestPlacement(t *testing.T) { if got := g14.Held(); !reflect.DeepEqual(got, []catalogue.Held{{Claim: "node-shell", Scope: "node", Node: "g14", Module: "zsh"}}) { t.Errorf("Held = %v", got) } + layered := PlacedModule{Module: "app", HasMesh: true, Layer: map[string]any{"expose": map[string]any{"80": "mesh"}}} + want := []catalogue.Layer{ + {From: catalogue.MeshWideLayer, Values: map[string]any{}}, + {From: "g14", Values: map[string]any{"mode": "x"}, Placement: map[string]any{"expose": map[string]any{"80": "mesh"}}}, + } + if got := layered.Layers("g14", map[string]any{"mode": "x"}); !reflect.DeepEqual(got, want) { + t.Errorf("Layers = %#v, want %#v", got, want) + } + if got := (PlacedModule{Module: "a"}).Layers("g14", map[string]any{"k": 1}); len(got) != 1 || got[0].From != "g14" { + t.Errorf("values alone make the node's layer: %#v", got) + } + if got := (PlacedModule{Module: "a", HasLayer: true}).Layers("g14", nil); len(got) != 1 || got[0].Values == nil { + t.Errorf("an empty layer of the node's own is kept, with empty values: %#v", got) + } + if got := (PlacedModule{Module: "a"}).Layers("g14", nil); got != nil { + t.Errorf("no layer and no value is no layer: %#v", got) + } empty := Placement{Node: "x"} if empty.ChosenPins() != nil || empty.Bound() != nil || empty.HostPorts() != nil || len(empty.Held()) != 0 { t.Error("an empty placement reads as something") From bdd4c7b87518a30b2935f0c53791f77484e9bbdf Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:07:20 +0100 Subject: [PATCH 03/42] =?UTF-8?q?The=20judgement=20of=20a=20move=20is=20a?= =?UTF-8?q?=20pure=20function=20in=20mesh-sdk,=20so=20a=20node=20agent=20j?= =?UTF-8?q?udges=20what=20it=20composed=20against=20what=20it=20runs=20(is?= =?UTF-8?q?sue=201126,=20ADR=200547=20=C2=A73,=20ADR=200377=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/delivery_test.go | 3 +- go/delivery/judge.go | 229 +++++++++++++++++++++++++++++++++++ go/delivery/judge_test.go | 68 +++++++++++ 3 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 go/delivery/judge.go create mode 100644 go/delivery/judge_test.go diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index f455eb3..9e6b68c 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -127,6 +127,7 @@ func testPulseKey(t *testing.T) { // 1131/AC4: each value encodes, decodes equal, and carries "v":1. func TestRoundTrip(t *testing.T) { + t.Run("phase and health refused", testPhaseHealthRefused) hasV1 := func(t *testing.T, b []byte) { t.Helper() var m map[string]any @@ -343,7 +344,7 @@ func TestUnknownVersion(t *testing.T) { } // 1131/AC6: a phase or health outside the lists is refused on encode and decode. -func TestPhaseHealthRefused(t *testing.T) { +func testPhaseHealthRefused(t *testing.T) { for _, p := range []Phase{PhaseBackingUp, PhaseApplying, PhaseChecking, PhaseRunning, PhaseFailed} { for _, h := range []Health{HealthOK, HealthFailed, HealthWaits, HealthUnknown} { if _, err := EncodeHave(Have{Phase: p, Health: h}); err != nil { diff --git a/go/delivery/judge.go b/go/delivery/judge.go new file mode 100644 index 0000000..f030401 --- /dev/null +++ b/go/delivery/judge.go @@ -0,0 +1,229 @@ +package delivery + +import ( + "bytes" + "encoding/json" + "path" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// The judgement of a move (ADR 0377 §3), as a pure function over two compositions of one module, so the node agent that +// composes its own modules judges what it would apply (novox/hq ADR 0547 §3; issue 1126's S8). Moved from +// mesh-controller cmd/mesh-controller/composed_judge.go at origin/main (the controller keeps its copy, unchanged, until +// it loses the lease, ADR 0547 §1): judged, onlyTheUserList and sameResources, with their helpers, are its code. +// +// ADR 0377 §3 over two compositions of one module: **recreates** when a container resource is changed, added or +// removed; **moves data** when, with no container changed, a directory or file resource is changed, added or removed +// that a container of the module mounts or that holds or lies within one of its directories (where a module keeps its +// data), when its `data` declaration changes, or, for a module that declares data, when one of its long-running +// processes changes; **neither** otherwise, and for a module the node has never run. + +// Judged is one move's judgement: what it does, its words, and whether only the bus's user list changes (ADR 0539 §7: +// a file change and a reload of the bus, no move of it, never asked). +type Judged struct { + Judgement Judgement + Words string + UserListOnly bool +} + +// JudgeMove judges moving a node's module from the composition it runs to the one composed for it. Running is nil when the +// node runs no composition of the module. A composition equal to the one running (resource for resource, each +// canonical) is unchanged: no move, and no ask (ADR 0547 §3). +func JudgeMove(running *Composed, composed Composed) Judged { + if composed.Module == "" { + return Judged{Judgement: JudgedNeither, Words: "the node's own composition"} + } + if running == nil { + return Judged{Judgement: JudgedNeither, Words: "the node runs no composition of " + composed.Module} + } + if running.Data == composed.Data && SameComposedResources(*running, composed) { + return Judged{Judgement: JudgedUnchanged, Words: "equal to the composition the node runs"} + } + if onlyTheUserList(*running, composed) { + return Judged{Judgement: JudgedNeither, Words: "only the bus's user list changes", UserListOnly: true} + } + j, words := judged(*running, composed) + return Judged{Judgement: j, Words: words} +} + +// busUsersSuffix ends the id of the bus module's user list as the composer writes it: `.bus-users`. +const busUsersSuffix = ".bus-users" + +// onlyTheUserList says two compositions of a module differ, and only in its file of the bus's user list. +func onlyTheUserList(from, to Composed) bool { + if from.Data != to.Data { + return false + } + before, after := judgedResourcesByID(from), judgedResourcesByID(to) + if len(before) != len(after) { + return false + } + changed := false + for id, now := range after { + was, ok := before[id] + if !ok { + return false + } + if bytes.Equal(was.raw, now.raw) { + continue + } + if !strings.HasSuffix(id, busUsersSuffix) || now.kind != string(declaration.TypeFile) { + return false + } + changed = true + } + return changed +} + +// judged applies ADR 0377 §3 to two compositions of one module. +func judged(from, to Composed) (Judgement, string) { + before, after := judgedResourcesByID(from), judgedResourcesByID(to) + if ids := changedOf(before, after, func(kind string) bool { return kind == string(declaration.TypeContainer) }); len(ids) > 0 { + return JudgedRecreates, "it recreates " + strings.Join(ids, ", ") + } + if from.Data != to.Data { + return JudgedMovesData, "its data declaration changes" + } + places := dataPlaces(before, after) + held := func(kind string) bool { + return kind == string(declaration.TypeDirectory) || kind == string(declaration.TypeFile) + } + var moved []string + for _, id := range changedOf(before, after, held) { + r := before[id] + if r == nil { + r = after[id] + } + if withinPlaces(r.path, places) { + moved = append(moved, id) + } + } + if to.Data || from.Data { + moved = append(moved, changedOf(before, after, func(kind string) bool { return kind == string(declaration.TypeProcess) })...) + } + if len(moved) > 0 { + sort.Strings(moved) + return JudgedMovesData, "it changes where its data is kept: " + strings.Join(moved, ", ") + } + return JudgedNeither, "it recreates no container and moves no data" +} + +// judgedResource is a resource as the judgement reads it: its kind, the path it acts on, what its containers mount, +// and its canonical bytes. +type judgedResource struct { + kind string + path string + mounts []string + raw []byte +} + +func judgedResourcesByID(c Composed) map[string]*judgedResource { + out := map[string]*judgedResource{} + for _, raw := range c.Resources { + var head struct { + ID string `json:"id"` + Type string `json:"type"` + Path string `json:"path"` + // A container's volumes, `:[:]`. + Volumes []string `json:"volumes"` + } + if err := json.Unmarshal(raw, &head); err != nil { + continue + } + r := &judgedResource{kind: head.Type, path: head.Path, raw: compactResource(raw)} + for _, v := range head.Volumes { + source, _, _ := strings.Cut(v, ":") + r.mounts = append(r.mounts, source) + } + out[head.ID] = r + } + return out +} + +// compactResource is a resource's bytes canonical: its keys sorted, its spacing dropped, its numbers as written, so two +// compositions that say the same compare equal however each was written. +func compactResource(raw []byte) []byte { + var v any + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + if err := d.Decode(&v); err != nil { + return raw + } + out, err := json.Marshal(v) + if err != nil { + return raw + } + return out +} + +// changedOf are the ids of the resources of the kinds counted that are changed, added or removed, sorted. +func changedOf(before, after map[string]*judgedResource, counted func(kind string) bool) []string { + var out []string + for id, now := range after { + if !counted(now.kind) { + continue + } + if was, ok := before[id]; !ok || !bytes.Equal(was.raw, now.raw) { + out = append(out, id) + } + } + for id, was := range before { + if _, ok := after[id]; !ok && counted(was.kind) { + out = append(out, id) + } + } + sort.Strings(out) + return out +} + +// dataPlaces are the paths where the module keeps what a move must not lose: what its containers mount, and its +// directories, before and after. +func dataPlaces(before, after map[string]*judgedResource) []string { + var out []string + for _, rs := range []map[string]*judgedResource{before, after} { + for _, r := range rs { + if r.kind == string(declaration.TypeContainer) { + out = append(out, r.mounts...) + } + if r.kind == string(declaration.TypeDirectory) && r.path != "" { + out = append(out, r.path) + } + } + } + return out +} + +// withinPlaces says p is one of places, or lies within one, or holds one. +func withinPlaces(p string, places []string) bool { + if p == "" { + return false + } + p = path.Clean(p) + for _, place := range places { + if place == "" || !strings.HasPrefix(place, "/") { + continue + } + place = path.Clean(place) + if p == place || strings.HasPrefix(p, place+"/") || strings.HasPrefix(place, p+"/") { + return true + } + } + return false +} + +// SameComposedResources says two compositions of a module hold the same resources in the same order, each compared +// canonical. +func SameComposedResources(a, b Composed) bool { + if len(a.Resources) != len(b.Resources) { + return false + } + for i := range a.Resources { + if !bytes.Equal(compactResource(a.Resources[i]), compactResource(b.Resources[i])) { + return false + } + } + return true +} diff --git a/go/delivery/judge_test.go b/go/delivery/judge_test.go new file mode 100644 index 0000000..4e61fd5 --- /dev/null +++ b/go/delivery/judge_test.go @@ -0,0 +1,68 @@ +package delivery + +import ( + "encoding/json" + "strings" + "testing" +) + +func composedOf(module string, c byte, data bool, resources ...string) Composed { + out := Composed{Node: "ace", Module: module, FP: SourcePrefix2 + strings.Repeat(string(c), 64), Data: data} + for _, r := range resources { + out.Resources = append(out.Resources, json.RawMessage(r)) + } + return out +} + +// The pairs mesh-controller's composed_judge_test.go captured (issue 1945 AC12), and the bus's user list, judged as the +// controller judges them (ADR 0377 §3, ADR 0539 §7), by the node agent's function (ADR 0547 §3, issue 1126 S8). +func TestJudgeMove(t *testing.T) { + const ( + container = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:1111111111111111111111111111111111111111111111111111111111111111","volumes":["/var/lib/mesh-mail/spool:/spool"]}` + container2 = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:2222222222222222222222222222222222222222222222222222222222222222","volumes":["/var/lib/mesh-mail/spool:/spool"]}` + spool = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}` + spoolMoved = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/srv/mesh-mail/spool"}` + tools = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"a"}` + tools2 = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"b"}` + config = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"a"}` + config2 = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"b"}` + process = `{"id":"mesh-mail.daemon","type":"process","module":"mesh-mail","exec":"/usr/bin/a"}` + process2 = `{"id":"mesh-mail.daemon","type":"process","module":"mesh-mail","exec":"/usr/bin/b"}` + users = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users","content":"a"}` + users2 = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users","content":"b"}` + natsConf = `{"id":"nats.conf","type":"file","module":"nats","path":"/etc/nats/conf","content":"a"}` + natsConf2 = `{"id":"nats.conf","type":"file","module":"nats","path":"/etc/nats/conf","content":"b"}` + ) + mail := func(c byte, data bool, rs ...string) *Composed { + m := composedOf("mesh-mail", c, data, rs...) + return &m + } + bus := func(rs ...string) *Composed { m := composedOf("nats", 'b', false, rs...); return &m } + for _, c := range []struct { + name string + from, to *Composed + want Judgement + userList bool + }{ + {"a new image", mail('a', true, container, spool, tools), mail('b', true, container2, spool, tools), JudgedRecreates, false}, + {"a container gone", mail('a', true, container, spool), mail('b', true, spool), JudgedRecreates, false}, + {"its spool moved", mail('a', true, container, spool), mail('b', true, container, spoolMoved), JudgedMovesData, false}, + {"a file in its spool", mail('a', true, container, spool, config), mail('b', true, container, spool, config2), JudgedMovesData, false}, + {"its data declaration", mail('a', false, container, spool), mail('b', true, container, spool), JudgedMovesData, false}, + {"a process of a module with data", mail('a', true, spool, process), mail('b', true, spool, process2), JudgedMovesData, false}, + {"a process of a module without", mail('a', false, process), mail('b', false, process2), JudgedNeither, false}, + {"its tools only", mail('a', true, container, spool, tools), mail('b', true, container, spool, tools2), JudgedNeither, false}, + {"never ran", nil, mail('b', true, container), JudgedNeither, false}, + {"equal, written otherwise", mail('a', true, `{"type":"directory","id":"mesh-mail.spool","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}`), mail('b', true, spool), JudgedUnchanged, false}, + {"only the user list", bus(natsConf, users), bus(natsConf, users2), JudgedNeither, true}, + {"the user list and more", bus(natsConf, users), bus(natsConf2, users2), JudgedNeither, false}, + } { + got := JudgeMove(c.from, *c.to) + if got.Judgement != c.want || got.UserListOnly != c.userList || got.Words == "" { + t.Errorf("%s: %+v, want %s (user list only %v)", c.name, got, c.want, c.userList) + } + } + if got := JudgeMove(nil, Composed{Node: "ace"}); got.Judgement != JudgedNeither { + t.Errorf("the node's own composition: %+v", got) + } +} From 9f7f50de8d1d91bb8f677ca04057ecdd7b2c4dd2 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:08:59 +0100 Subject: [PATCH 04/42] =?UTF-8?q?A=20merged=20message=20of=20the=20catalog?= =?UTF-8?q?ue=20carries=20its=20placement=20files,=20so=20the=20indexer=20?= =?UTF-8?q?reads=20placement=20at=20the=20merge=20without=20asking=20the?= =?UTF-8?q?=20forge=20again=20(issue=201126,=20ADR=200549=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/forge_inputs_test.go | 12 ++++++++---- go/delivery/source.go | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/go/delivery/forge_inputs_test.go b/go/delivery/forge_inputs_test.go index e027ff7..802aa77 100644 --- a/go/delivery/forge_inputs_test.go +++ b/go/delivery/forge_inputs_test.go @@ -9,9 +9,10 @@ const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" func mergedWithInputs() Forge { return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0", - Trees: map[string]string{"a": "t1", "b": "t2"}, - Manifests: map[string]string{"a": oddManifest}, - Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}} + Trees: map[string]string{"a": "t1", "b": "t2"}, + Manifests: map[string]string{"a": oddManifest}, + Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}, + Placements: map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`}, PlacementsRead: true} } // 1442/AC1 @@ -21,7 +22,7 @@ func TestForgeMergedCarriesManifestsAndLocks(t *testing.T) { if err != nil { t.Fatal(err) } - for _, frag := range []string{`"manifests":{"a":`, `"locks":{"a":{"go.sum":`} { + for _, frag := range []string{`"manifests":{"a":`, `"locks":{"a":{"go.sum":`, `"placements":{"ace":`, `"placements-read":true`} { if !strings.Contains(string(data), frag) { t.Errorf("the encoding lacks %s: %s", frag, data) } @@ -47,6 +48,9 @@ func TestForgeMergedRefusesBadInputs(t *testing.T) { "locks outside trees": {func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} }, `"yy"`}, "empty manifest": {func(f *Forge) { f.Manifests["b"] = "" }, `"b"`}, "empty lock hash": {func(f *Forge) { f.Locks["a"]["go.sum"] = "" }, `"a"`}, + "placements unsaid": {func(f *Forge) { f.PlacementsRead = false }, "placements-read"}, + "placement of no node": {func(f *Forge) { f.Placements["a.b"] = "{}" }, "names no node"}, + "placements on a pull": {func(f *Forge) { f.Kind, f.Action, f.Merge, f.Root, f.Trees = ForgePull, PullOpened, "", "", nil }, "only a merged message"}, } { f := mergedWithInputs() tc.mut(&f) diff --git a/go/delivery/source.go b/go/delivery/source.go index 535f771..71b4de1 100644 --- a/go/delivery/source.go +++ b/go/delivery/source.go @@ -537,6 +537,11 @@ type Forge struct { Act string `json:"act,omitempty"` Words string `json:"words,omitempty"` Open []OpenPull `json:"open,omitempty"` + // Placements are, on a merged message of the catalogue repository, every placement file of the merge commit, the + // bytes exact, by node (PlacementFile; novox/hq ADR 0549 §2), and PlacementsRead says they were read, so a catalogue + // whose last placement file was removed is told from a message that read none (older writers, another repository). + Placements map[string]string `json:"placements,omitempty"` + PlacementsRead bool `json:"placements-read,omitempty"` } func (f Forge) check() error { @@ -549,6 +554,17 @@ func (f Forge) check() error { if len(f.Open) > 0 { return fmt.Errorf("delivery: only an open message names open pull requests") } + if (len(f.Placements) > 0 || f.PlacementsRead) && f.Kind != ForgeMerged { + return fmt.Errorf("delivery: only a merged message carries placement files") + } + if len(f.Placements) > 0 && !f.PlacementsRead { + return fmt.Errorf("delivery: a message with placement files must say it read them (placements-read)") + } + for node := range f.Placements { + if err := checkName("node", node); err != nil { + return fmt.Errorf("delivery: a placement file names no node: %w", err) + } + } if err := checkRepository(f.Repository); err != nil { return err } From 5db53e738f3aaef061dd850f4c3c960ce42a0299 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:10:25 +0100 Subject: [PATCH 05/42] =?UTF-8?q?Name=20the=20catalogue=20repository=20who?= =?UTF-8?q?se=20placement=20files=20are=20read=20(issue=201126,=20ADR=2005?= =?UTF-8?q?49=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/placement.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/delivery/placement.go b/go/delivery/placement.go index 3659b3b..482ff7d 100644 --- a/go/delivery/placement.go +++ b/go/delivery/placement.go @@ -28,6 +28,10 @@ const ( PlacementChangedEvent = "placement-changed" placementWord = "placement" + + // CatalogueRepository is the repository whose placement files are the mesh's placement (ADR 0549 §1): the forge bridge + // reads them at each of its merges, and no other repository's. + CatalogueRepository = "novox/mesh-catalog" ) // PlacementKey is the key, in PlacementBucket, of one node's placement: `placement.`. From a431746c0e8c21700c2d8bc235c2aa3a410c515f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:17:59 +0100 Subject: [PATCH 06/42] What a person decides of a node lives in its placement file and what the machine observes in its record; two nodes on one address or two hubs are refused (issue 1126, K2) --- go/compose/frombus/frombus.go | 26 +++++++++++--------- go/compose/frombus/frombus_test.go | 16 ++++++------ go/delivery/noderecord.go | 36 ++++++++------------------- go/delivery/noderecord_test.go | 18 ++++++-------- go/delivery/placement.go | 39 +++++++++++++++++++++++++++++- go/delivery/placement_test.go | 7 ++++-- 6 files changed, 84 insertions(+), 58 deletions(-) diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 2e96c5b..c698795 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -9,8 +9,10 @@ // **What it reads, and from whom.** // - which modules run on each node, with their pins, holdings, host ports, bindings and settings placement: the // indexer's state `placement`, one key per node (ADR 0549 §2); -// - what each node knows of itself (its place on the private network, capabilities, accounts, keys, networks, -// tunnel, and on the hub the adopted tunnel and its carried peers): the node-engine's state `records`; +// - what a person decided of each node (its endpoint, site, hub, address on the private network, public domain and +// accounts): the same placement key, Placement.Decided; +// - what each node observes of itself (its tunnel key, capabilities, keys, networks, the tunnel it found, and on the +// hub the adopted tunnel and its carried peers): the node-engine's state `records`; // - every module's manifest: the indexer's `modules.` and its manifests by digest, or the digest the // composing node's want names where it runs another build (Scope.Kept); // - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the @@ -170,12 +172,12 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if in.Tunnels == nil { in.Tunnels = map[string]compose.FoundTunnel{} } - in.Tunnels[node] = compose.FoundTunnel{Tunnel: tunnelOf(*rec.Tunnel), NodeAdopted: rec.Adopted} - if rec.Place.Hub { + in.Tunnels[node] = compose.FoundTunnel{Tunnel: tunnelOf(*rec.Tunnel), NodeAdopted: p.Decided.Adopted} + if p.Decided.Hub { in.Adopted = &compose.AdoptedTunnel{Tunnel: tunnelOf(*rec.Tunnel), Hub: node, Adopted: rec.HubAdopted} } } - if rec.Place.Hub { + if p.Decided.Hub { for _, c := range rec.Carried { in.CarriedPeers = append(in.CarriedPeers, compose.CarriedPeer{PublicKey: c.PublicKey, Address: c.Address, EnrolledAs: c.EnrolledAs, Named: c.Named}) @@ -204,15 +206,17 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s return compose.Compositions(in, s.Node, fps, compose.Reading) } -// nodeOf is the composition's node from its record and its placement. +// nodeOf is the composition's node from its record (what the machine observes) and its placement (what a person +// decided of it, and where its modules run). func nodeOf(rec delivery.NodeRecord, p delivery.Placement) compose.Node { + d := p.Decided n := compose.Node{ Name: rec.Node, - Place: compose.Place{Key: rec.Place.Key, Endpoint: rec.Place.Endpoint, Site: rec.Place.Site, Hub: rec.Place.Hub, Address: rec.Place.Address}, - Assigned: p.Assigned(), PublicDomain: rec.PublicDomain, - Account: rec.Account, AccountHome: rec.AccountHome, AgentAccount: rec.AgentAccount, AgentAccountHome: rec.AgentHome, - Adopted: rec.Adopted, Pins: p.ChosenPins(), Bound: p.Bound(), Ports: p.HostPorts(), SealingKey: rec.SealingKey, - Taken: rec.Taken, OutwardLinks: rec.OutwardLinks, Networks: rec.Networks, NetworksSaid: rec.NetworksSaid, + Place: compose.Place{Key: rec.TunnelKey, Endpoint: d.Endpoint, Site: d.Site, Hub: d.Hub, Address: d.Address}, + Assigned: p.Assigned(), PublicDomain: d.PublicDomain, + Account: d.Account, AccountHome: d.AccountHome, AgentAccount: d.AgentAccount, AgentAccountHome: d.AgentHome, + Adopted: d.Adopted, Taken: d.Taken, Pins: p.ChosenPins(), Bound: p.Bound(), Ports: p.HostPorts(), SealingKey: rec.SealingKey, + OutwardLinks: rec.OutwardLinks, Networks: rec.Networks, NetworksSaid: rec.NetworksSaid, ServingKey: rec.ServingKey, Certificate: rec.Certificate, Contract: rec.Contract, Stated: rec.Contract > 0, } if len(rec.Capabilities) > 0 { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 93d8f1e..8246eb5 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -71,17 +71,17 @@ func TestInput(t *testing.T) { mk := func() *bus { return &bus{ placements: map[string]delivery.Placement{ - "anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: overlay.Name}}, - Holds: []delivery.Holding{{Claim: "mesh-terminal", Scope: "mesh", Module: "xterm"}}}, - "laptop": {Node: "laptop", Modules: []delivery.PlacedModule{{Module: overlay.Name}, - {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}}}, + "anchor": {Node: "anchor", Decided: delivery.NodeDecided{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"}, + Modules: []delivery.PlacedModule{{Module: overlay.Name}}, + Holds: []delivery.Holding{{Claim: "mesh-terminal", Scope: "mesh", Module: "xterm"}}}, + "laptop": {Node: "laptop", Decided: delivery.NodeDecided{Endpoint: "laptop.example:51820", Site: "here", Address: "10.77.0.2"}, + Modules: []delivery.PlacedModule{{Module: overlay.Name}, + {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}}}, }, records: map[string]delivery.NodeRecord{ - "anchor": {Node: "anchor", Place: delivery.NodePlace{Key: keys["anchor"], Endpoint: "anchor.example:51820", - Site: "here", Hub: true, Address: "10.77.0.1"}, Capabilities: caps, Tunnel: tunnel, Contract: 2, + "anchor": {Node: "anchor", TunnelKey: keys["anchor"], Capabilities: caps, Tunnel: tunnel, Contract: 2, Carried: []delivery.RecordCarriedPeer{{PublicKey: "peer", Address: "10.77.0.9", EnrolledAs: "phone"}}}, - "laptop": {Node: "laptop", Place: delivery.NodePlace{Key: keys["laptop"], Endpoint: "laptop.example:51820", - Site: "here", Address: "10.77.0.2"}, Capabilities: caps}, + "laptop": {Node: "laptop", TunnelKey: keys["laptop"], Capabilities: caps}, }, modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "unread": {}}, manifests: map[string][]byte{digest: xterm}, diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go index 72c8ac0..b159d42 100644 --- a/go/delivery/noderecord.go +++ b/go/delivery/noderecord.go @@ -45,16 +45,6 @@ func ParseNodeRecordKey(key string) (node string, err error) { return node, nil } -// NodePlace is where the node is on the private network: its tunnel key (public), endpoint, site, whether it is the -// hub, and its address (compose.Place). -type NodePlace struct { - Key string `json:"key,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Site string `json:"site,omitempty"` - Hub bool `json:"hub,omitempty"` - Address string `json:"address,omitempty"` -} - // RecordTunnel is a tunnel found on the node, as compose.Tunnel holds it: public keys and addresses only. type RecordTunnel struct { Interface string `json:"interface"` @@ -82,21 +72,17 @@ type RecordCarriedPeer struct { Named string `json:"named,omitempty"` } -// NodeRecord is one node's record. Tunnel is the tunnel found on it whose key is its private-network key; on the hub -// alone, Adopted says whether the hub took its found tunnel over and Carried is every peer of it the hub carries, so -// the tunnels and the carried peers are each one node's own and need no other home. +// NodeRecord is one node's record: what the machine observes of itself (issue 1126's K2: what a person decides lives +// in the catalogue's placement file, Placement.Decided; what the machine observes lives here). TunnelKey is the public +// key of its private-network tunnel; Tunnel the tunnel found on it whose key that is; on the hub alone, HubAdopted says +// whether the hub took its found tunnel over and Carried is every peer of it the hub carries, so the tunnels and the +// carried peers are each one node's own and need no other home. Certificate is the one the mesh issued for its +// internal name, public, as the node holds it. type NodeRecord struct { V int `json:"v"` Node string `json:"node"` - Place NodePlace `json:"place"` + TunnelKey string `json:"tunnel-key,omitempty"` Capabilities []string `json:"capabilities,omitempty"` - PublicDomain string `json:"public-domain,omitempty"` - Account string `json:"account,omitempty"` - AccountHome string `json:"account-home,omitempty"` - AgentAccount string `json:"agent-account,omitempty"` - AgentHome string `json:"agent-account-home,omitempty"` - Adopted bool `json:"adopted,omitempty"` - Taken []string `json:"taken,omitempty"` SealingKey string `json:"sealing-key,omitempty"` ServingKey string `json:"serving-key,omitempty"` Certificate string `json:"certificate,omitempty"` @@ -113,9 +99,6 @@ func (r NodeRecord) check() error { if err := checkName("node", r.Node); err != nil { return err } - if !r.Place.Hub && (r.HubAdopted || len(r.Carried) > 0) { - return fmt.Errorf("delivery: the record of %s carries peers or an adopted tunnel, which only the hub's does", r.Node) - } if r.HubAdopted && r.Tunnel == nil { return fmt.Errorf("delivery: the record of %s says the hub adopted its tunnel, and names none", r.Node) } @@ -136,8 +119,9 @@ func EncodeNodeRecord(r NodeRecord) ([]byte, error) { return json.Marshal(r) } -// DecodeNodeRecord reads a node record; it refuses an unknown version (ErrUnknownVersion), a node that is not a name, -// carried peers or an adopted tunnel on a node that is not the hub, and an empty capability. +// DecodeNodeRecord reads a node record; it refuses an unknown version (ErrUnknownVersion), a node that is not a name, an +// adopted tunnel with no tunnel, and an empty capability. Whether the node is the hub is the catalogue's (Placement.Decided): +// a reader takes carried peers and an adopted tunnel only from the hub's record. func DecodeNodeRecord(data []byte) (NodeRecord, error) { var r NodeRecord if err := decode(data, &r); err != nil { diff --git a/go/delivery/noderecord_test.go b/go/delivery/noderecord_test.go index e882f76..c816e71 100644 --- a/go/delivery/noderecord_test.go +++ b/go/delivery/noderecord_test.go @@ -13,8 +13,8 @@ import ( // check (ADR 0201 §13) refuses such a field in a state every node reads. var secretNamed = regexp.MustCompile(`(?i)(secret|passw|token|api[-_]?key|private[-_]?key|credential|bearer|cookie|salt|signing)`) -// Issue 1126 (L1 as narrowed by ADR 0549): a node's record keys, reads back, refuses what only the hub may carry, and -// names no field like a secret. +// Issue 1126 (K2): a node's record keys, reads back, refuses an adopted tunnel it does not name, and neither it nor the +// placement file names a field like a secret. func TestNodeRecord(t *testing.T) { key, err := NodeRecordKey("ace") if err != nil || key != "record.ace" || NodeRecordsBucket != "node-engine_records" { @@ -29,11 +29,9 @@ func TestNodeRecord(t *testing.T) { } } - hub := NodeRecord{Node: "novox", Place: NodePlace{Key: "k", Endpoint: "192.0.2.1:51820", Hub: true, Address: "10.0.0.1"}, - Capabilities: []string{"docker"}, PublicDomain: "example.org", Account: "op", AccountHome: "/home/op", - AgentAccount: "agent", AgentHome: "/var/lib/agent", SealingKey: "age1x", ServingKey: "pub", Certificate: "cert", - OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, - NetworksSaid: true, Contract: 3, Taken: []string{"x"}, Adopted: true, + hub := NodeRecord{Node: "novox", TunnelKey: "k", Capabilities: []string{"docker"}, SealingKey: "age1x", ServingKey: "pub", + Certificate: "cert", OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, + NetworksSaid: true, Contract: 3, Tunnel: &RecordTunnel{Interface: "wg0", PublicKey: "k", Peers: []RecordTunnelPeer{{PublicKey: "p"}}}, HubAdopted: true, Carried: []RecordCarriedPeer{{PublicKey: "p", EnrolledAs: "phone"}}} raw, err := EncodeNodeRecord(hub) @@ -46,9 +44,7 @@ func TestNodeRecord(t *testing.T) { } for name, r := range map[string]NodeRecord{ - "carried off the hub": {Node: "ace", Carried: []RecordCarriedPeer{{PublicKey: "p"}}}, - "adopted off the hub": {Node: "ace", HubAdopted: true}, - "adopted with no tunnel": {Node: "novox", Place: NodePlace{Hub: true}, HubAdopted: true}, + "adopted with no tunnel": {Node: "novox", HubAdopted: true}, "empty capability": {Node: "ace", Capabilities: []string{""}}, "node that is not a name": {Node: "a.b"}, } { @@ -76,4 +72,6 @@ func TestNodeRecord(t *testing.T) { } } walk(reflect.TypeOf(NodeRecord{}), "NodeRecord") + // And the placement file, which holds what a person decides of a node (K2): accounts by name, never a credential. + walk(reflect.TypeOf(Placement{}), "Placement") } diff --git a/go/delivery/placement.go b/go/delivery/placement.go index 482ff7d..4f48a1b 100644 --- a/go/delivery/placement.go +++ b/go/delivery/placement.go @@ -109,11 +109,33 @@ type Placement struct { V int `json:"v"` Node string `json:"node"` Revision string `json:"revision,omitempty"` + Decided NodeDecided `json:"decided"` Modules []PlacedModule `json:"modules"` Pins map[string]Provider `json:"pins,omitempty"` Holds []Holding `json:"holds,omitempty"` } +// NodeDecided is what a person decides of the node, kept in its placement file (issue 1126's K2: what a person decides +// lives in the catalogue, what the machine observes lives in its record, NodeRecord): where it is dialled (endpoint) and +// its site, whether it is the private network's hub, its address on the private network, the domain its routed names +// are composed under, its operator and agent logins by name and their homes, whether it is adopted rather than +// converged, and the modules taken on an adopted node. The address is a decision written once, at enrolment, by a +// catalogue pull request: nothing allocates it at run time, and CheckPlacements refuses two nodes with one address. +// No secret and no key material: accounts are names, never credentials. +type NodeDecided struct { + Endpoint string `json:"endpoint,omitempty"` + Site string `json:"site,omitempty"` + Hub bool `json:"hub,omitempty"` + Address string `json:"address,omitempty"` + PublicDomain string `json:"public-domain,omitempty"` + Account string `json:"account,omitempty"` + AccountHome string `json:"account-home,omitempty"` + AgentAccount string `json:"agent-account,omitempty"` + AgentHome string `json:"agent-account-home,omitempty"` + Adopted bool `json:"adopted,omitempty"` + Taken []string `json:"taken,omitempty"` +} + func (p Placement) check() error { if err := checkName("node", p.Node); err != nil { return err @@ -326,7 +348,8 @@ func (p Placement) Held() []catalogue.Held { // CheckPlacements is the catalogue's own check of every placement file (ADR 0549 §1), over the placements by node, the // modules the catalogue holds and the nodes enrolled. It returns every refusal, in node order, never only the first: a // placement naming a node not enrolled or a module the catalogue does not hold; two modules of one node on the same host -// port; a pin, binding or holding naming a provider that is not placed where it says; a seat of scope mesh held twice. +// port; a pin, binding or holding naming a provider that is not placed where it says; a seat of scope mesh held twice; +// two nodes with one address on the private network; more than one hub. // A host port a manifest declares without a number and a requirement left unbound are the composition's to find (its // ForeseenPorts and Unbound), which the catalogue's composition check reads. func CheckPlacements(placements map[string]Placement, held func(module string) bool, enrolled func(node string) bool) []string { @@ -349,11 +372,22 @@ func CheckPlacements(placements map[string]Placement, held func(module string) b return false } meshSeats := map[string]string{} + addresses := map[string]string{} + var hubs []string for _, node := range nodes { p := placements[node] if p.Node != node { refusals = append(refusals, fmt.Sprintf("the placement file of %s names the node %q", node, p.Node)) } + if a := strings.TrimSpace(p.Decided.Address); a != "" { + if other, twice := addresses[a]; twice { + refusals = append(refusals, fmt.Sprintf("%s and %s both have the address %s on the private network", other, node, a)) + } + addresses[a] = node + } + if p.Decided.Hub { + hubs = append(hubs, node) + } if !enrolled(node) { refusals = append(refusals, fmt.Sprintf("%s is not an enrolled node", node)) } @@ -391,6 +425,9 @@ func CheckPlacements(placements map[string]Placement, held func(module string) b meshSeats[seat] = node } } + if len(hubs) > 1 { + refusals = append(refusals, fmt.Sprintf("the private network has one hub, and %s are each placed as it", strings.Join(hubs, ", "))) + } return refusals } diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index 40c6eb3..3993556 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -40,6 +40,7 @@ func TestPlacement(t *testing.T) { }) g14 := Placement{Node: "g14", Revision: "abc", + Decided: NodeDecided{Endpoint: "g14.example:51820", Site: "home", Address: "10.77.0.2", Account: "op", Taken: []string{"zsh"}}, Modules: []PlacedModule{ {Module: "zsh"}, {Module: "app", Ports: map[string]int{"8080": 18080}, Bindings: map[string]Provider{"postgres": {Node: "novox", Module: "postgres"}}}, @@ -47,7 +48,7 @@ func TestPlacement(t *testing.T) { Pins: map[string]Provider{"shell": {Node: "g14", Module: "zsh"}}, Holds: []Holding{{Claim: "node-shell", Scope: "node", Module: "zsh"}}, } - novox := Placement{Node: "novox", Modules: []PlacedModule{{Module: "postgres", Ports: map[string]int{"5432": 5432}}}, + novox := Placement{Node: "novox", Decided: NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []PlacedModule{{Module: "postgres", Ports: map[string]int{"5432": 5432}}}, Holds: []Holding{{Claim: "mesh-store", Scope: "mesh", Module: "postgres"}}} t.Run("value", func(t *testing.T) { @@ -153,10 +154,12 @@ func TestPlacement(t *testing.T) { clash.Pins = map[string]Provider{"shell": {Node: "g14", Module: "bash"}} twice := novox twice.Node = "stranger" + twice.Decided.Address = "10.77.0.2" r := CheckPlacements(map[string]Placement{"g14": clash, "novox": novox, "stranger": twice}, held, enrolled) for _, want := range []string{"holds no module ghost", "ghost and app are both published on host port 18080", "bound to ace/postgres", "pin of shell names g14/bash", "stranger is not an enrolled node", - "mesh-store, held once for the mesh, is held on novox and stranger"} { + "mesh-store, held once for the mesh, is held on novox and stranger", + "g14 and stranger both have the address 10.77.0.2", "novox, stranger are each placed as it"} { found := false for _, line := range r { found = found || strings.Contains(line, want) From 5b1e990eb4bb0fa56e263671e375f482fa419667 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:26:11 +0100 Subject: [PATCH 07/42] The tunnel found at adoption and the names of the hub's peers are kept in the placement; whether the hub took it over and who enrolled under a peer's key are read from placement and records, as the controller reads them (issue 1126, ADR 0555) --- go/compose/frombus/frombus.go | 55 +++++++++++++++++++++++------- go/compose/frombus/frombus_test.go | 20 ++++++----- go/delivery/noderecord.go | 34 ++++++------------ go/delivery/noderecord_test.go | 9 ++--- go/delivery/placement.go | 7 +++- 5 files changed, 72 insertions(+), 53 deletions(-) diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index c698795..94aa253 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -11,8 +11,9 @@ // indexer's state `placement`, one key per node (ADR 0549 §2); // - what a person decided of each node (its endpoint, site, hub, address on the private network, public domain and // accounts): the same placement key, Placement.Decided; -// - what each node observes of itself (its tunnel key, capabilities, keys, networks, the tunnel it found, and on the -// hub the adopted tunnel and its carried peers): the node-engine's state `records`; +// - what each node observes of itself (its tunnel key, capabilities, keys, certificate, links and networks): the +// node-engine's state `records`; the tunnel found at a node's adoption and the names of the hub's peers are decided, +// in its placement, and whether the hub took the tunnel over and who enrolled under a peer's key are read from both; // - every module's manifest: the indexer's `modules.` and its manifests by digest, or the digest the // composing node's want names where it runs another build (Scope.Kept); // - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the @@ -28,6 +29,7 @@ import ( "context" "encoding/json" "fmt" + "net/netip" "sort" "strings" @@ -168,22 +170,24 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } in.Nodes = append(in.Nodes, n) in.Holdings = append(in.Holdings, p.Held()...) - if rec.Tunnel != nil { - if in.Tunnels == nil { - in.Tunnels = map[string]compose.FoundTunnel{} + // A tunnel found at adoption is the node's private network's when the node's tunnel key is the tunnel's (the + // controller's inv.Tunnels); the hub's found tunnel is the mesh's adopted one whether or not the hub took it over + // (inv.AdoptedTunnel), and its peers are carried once it did (inv.CarriedPeers). + if t := p.Decided.Tunnel; t != nil { + if rec.TunnelKey == t.PublicKey { + if in.Tunnels == nil { + in.Tunnels = map[string]compose.FoundTunnel{} + } + in.Tunnels[node] = compose.FoundTunnel{Tunnel: tunnelOf(*t), NodeAdopted: p.Decided.Adopted} } - in.Tunnels[node] = compose.FoundTunnel{Tunnel: tunnelOf(*rec.Tunnel), NodeAdopted: p.Decided.Adopted} if p.Decided.Hub { - in.Adopted = &compose.AdoptedTunnel{Tunnel: tunnelOf(*rec.Tunnel), Hub: node, Adopted: rec.HubAdopted} - } - } - if p.Decided.Hub { - for _, c := range rec.Carried { - in.CarriedPeers = append(in.CarriedPeers, compose.CarriedPeer{PublicKey: c.PublicKey, Address: c.Address, - EnrolledAs: c.EnrolledAs, Named: c.Named}) + in.Adopted = &compose.AdoptedTunnel{Tunnel: tunnelOf(*t), Hub: node, Adopted: rec.TunnelKey == t.PublicKey} } } } + if in.Adopted != nil && in.Adopted.Adopted { + in.CarriedPeers = carriedPeers(in.Adopted.Tunnel, placements[in.Adopted.Hub].Decided.PeerNames, records) + } if len(missing) > 0 { return compose.Input{}, fmt.Errorf("frombus: modules are placed on %s, whose agent has written no record yet", strings.Join(missing, ", ")) @@ -237,6 +241,31 @@ func tunnelOf(t delivery.RecordTunnel) compose.Tunnel { return out } +// carriedPeers is every peer of the hub's adopted tunnel, by address, each with the node that enrolled under its key +// where one has (a node whose tunnel key is the peer's) and the name a person gave it (inv.CarriedPeers). +func carriedPeers(t compose.Tunnel, names map[string]string, records map[string]delivery.NodeRecord) []compose.CarriedPeer { + byKey := map[string]string{} + for node, rec := range records { + if rec.TunnelKey != "" { + byKey[rec.TunnelKey] = node + } + } + var out []compose.CarriedPeer + for _, p := range t.Peers { + address, _, _ := strings.Cut(p.Address, "/") + out = append(out, compose.CarriedPeer{PublicKey: p.PublicKey, Address: address, EnrolledAs: byKey[p.PublicKey], Named: names[p.PublicKey]}) + } + sort.SliceStable(out, func(i, j int) bool { + a, errA := netip.ParseAddr(out[i].Address) + b, errB := netip.ParseAddr(out[j].Address) + if errA != nil || errB != nil { + return out[i].Address < out[j].Address + } + return a.Less(b) + }) + return out +} + // nodesOf is every node named by a placement or a record, by name: the order the controller reads them in. func nodesOf(placements map[string]delivery.Placement, records map[string]delivery.NodeRecord) []string { seen := map[string]bool{} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 8246eb5..8e93364 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -67,11 +67,12 @@ func TestInput(t *testing.T) { keys := map[string]string{"anchor": publicKey(t), "laptop": publicKey(t)} caps := []string{"container-runtime", "systemd", "wireguard"} tunnel := &delivery.RecordTunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", - Peers: []delivery.RecordTunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}}} + Peers: []delivery.RecordTunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}, {PublicKey: keys["laptop"], Address: "10.77.0.2/32"}}} mk := func() *bus { return &bus{ placements: map[string]delivery.Placement{ - "anchor": {Node: "anchor", Decided: delivery.NodeDecided{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"}, + "anchor": {Node: "anchor", Decided: delivery.NodeDecided{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1", + Tunnel: tunnel, PeerNames: map[string]string{"peer": "phone"}}, Modules: []delivery.PlacedModule{{Module: overlay.Name}}, Holds: []delivery.Holding{{Claim: "mesh-terminal", Scope: "mesh", Module: "xterm"}}}, "laptop": {Node: "laptop", Decided: delivery.NodeDecided{Endpoint: "laptop.example:51820", Site: "here", Address: "10.77.0.2"}, @@ -79,8 +80,7 @@ func TestInput(t *testing.T) { {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}}}, }, records: map[string]delivery.NodeRecord{ - "anchor": {Node: "anchor", TunnelKey: keys["anchor"], Capabilities: caps, Tunnel: tunnel, Contract: 2, - Carried: []delivery.RecordCarriedPeer{{PublicKey: "peer", Address: "10.77.0.9", EnrolledAs: "phone"}}}, + "anchor": {Node: "anchor", TunnelKey: keys["anchor"], Capabilities: caps, Contract: 2}, "laptop": {Node: "laptop", TunnelKey: keys["laptop"], Capabilities: caps}, }, modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "unread": {}}, @@ -101,7 +101,7 @@ func TestInput(t *testing.T) { } capsSet := map[string]bool{"container-runtime": true, "systemd": true, "wireguard": true} tun := compose.Tunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", - Peers: []compose.TunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}}} + Peers: []compose.TunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}, {PublicKey: keys["laptop"], Address: "10.77.0.2/32"}}} want := compose.Input{ Catalogue: map[string]catalogue.Manifest{overlay.Name: private, "xterm": parsed}, Nodes: []compose.Node{ @@ -112,10 +112,12 @@ func TestInput(t *testing.T) { Ports: map[string]map[int]int{"xterm": {80: 8080}}, Settings: map[string][]catalogue.Layer{"xterm": {{From: "laptop", Values: map[string]any{"font": "mono"}}}}}, }, - Holdings: []catalogue.Held{{Claim: "mesh-terminal", Scope: "mesh", Node: "anchor", Module: "xterm"}}, - Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, - Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor"}, - CarriedPeers: []compose.CarriedPeer{{PublicKey: "peer", Address: "10.77.0.9", EnrolledAs: "phone"}}, + Holdings: []catalogue.Held{{Claim: "mesh-terminal", Scope: "mesh", Node: "anchor", Module: "xterm"}}, + Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, + Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, + // The controller's order is by address; laptop enrolled under its peer's key, and a person named the other. + CarriedPeers: []compose.CarriedPeer{{PublicKey: keys["laptop"], Address: "10.77.0.2", EnrolledAs: "laptop"}, + {PublicKey: "peer", Address: "10.77.0.9", Named: "phone"}}, } b := mk() diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go index b159d42..3b662b0 100644 --- a/go/delivery/noderecord.go +++ b/go/delivery/noderecord.go @@ -45,7 +45,9 @@ func ParseNodeRecordKey(key string) (node string, err error) { return node, nil } -// RecordTunnel is a tunnel found on the node, as compose.Tunnel holds it: public keys and addresses only. +// RecordTunnel is a tunnel found on a node when it was adopted, as compose.Tunnel holds it: public keys and addresses +// only. After the node took it over the tunnel is no longer there to observe, so it is kept as what was decided at +// adoption, in the node's placement (NodeDecided.Tunnel). type RecordTunnel struct { Interface string `json:"interface"` Unit string `json:"unit,omitempty"` @@ -64,20 +66,11 @@ type RecordTunnelPeer struct { Address string `json:"address,omitempty"` } -// RecordCarriedPeer is a peer of the tunnel the hub adopted, which the hub carries (compose.CarriedPeer). -type RecordCarriedPeer struct { - PublicKey string `json:"public-key"` - Address string `json:"address,omitempty"` - EnrolledAs string `json:"enrolled-as,omitempty"` - Named string `json:"named,omitempty"` -} - -// NodeRecord is one node's record: what the machine observes of itself (issue 1126's K2: what a person decides lives -// in the catalogue's placement file, Placement.Decided; what the machine observes lives here). TunnelKey is the public -// key of its private-network tunnel; Tunnel the tunnel found on it whose key that is; on the hub alone, HubAdopted says -// whether the hub took its found tunnel over and Carried is every peer of it the hub carries, so the tunnels and the -// carried peers are each one node's own and need no other home. Certificate is the one the mesh issued for its -// internal name, public, as the node holds it. +// NodeRecord is one node's record: what the machine observes of itself (issue 1126, ADR 0555: what a person decides +// lives in the catalogue's placement file, Placement.Decided; what the machine observes lives here). TunnelKey is the +// public key of its private-network tunnel; Certificate the one the mesh issued for its internal name, public, as the +// node holds it. The tunnel found at adoption and the names given to its peers are decisions kept in the placement; +// whether the hub took its found tunnel over, and which node enrolled under a carried peer's key, are read from both. type NodeRecord struct { V int `json:"v"` Node string `json:"node"` @@ -90,18 +83,12 @@ type NodeRecord struct { Networks []catalogue.AttachedNetwork `json:"networks,omitempty"` NetworksSaid bool `json:"networks-said,omitempty"` Contract int `json:"contract,omitempty"` - Tunnel *RecordTunnel `json:"tunnel,omitempty"` - HubAdopted bool `json:"hub-adopted,omitempty"` - Carried []RecordCarriedPeer `json:"carried,omitempty"` } func (r NodeRecord) check() error { if err := checkName("node", r.Node); err != nil { return err } - if r.HubAdopted && r.Tunnel == nil { - return fmt.Errorf("delivery: the record of %s says the hub adopted its tunnel, and names none", r.Node) - } for _, c := range r.Capabilities { if c == "" { return fmt.Errorf("delivery: the record of %s names an empty capability", r.Node) @@ -119,9 +106,8 @@ func EncodeNodeRecord(r NodeRecord) ([]byte, error) { return json.Marshal(r) } -// DecodeNodeRecord reads a node record; it refuses an unknown version (ErrUnknownVersion), a node that is not a name, an -// adopted tunnel with no tunnel, and an empty capability. Whether the node is the hub is the catalogue's (Placement.Decided): -// a reader takes carried peers and an adopted tunnel only from the hub's record. +// DecodeNodeRecord reads a node record; it refuses an unknown version (ErrUnknownVersion), a node that is not a name, and +// an empty capability. func DecodeNodeRecord(data []byte) (NodeRecord, error) { var r NodeRecord if err := decode(data, &r); err != nil { diff --git a/go/delivery/noderecord_test.go b/go/delivery/noderecord_test.go index c816e71..e69e24f 100644 --- a/go/delivery/noderecord_test.go +++ b/go/delivery/noderecord_test.go @@ -13,8 +13,8 @@ import ( // check (ADR 0201 §13) refuses such a field in a state every node reads. var secretNamed = regexp.MustCompile(`(?i)(secret|passw|token|api[-_]?key|private[-_]?key|credential|bearer|cookie|salt|signing)`) -// Issue 1126 (K2): a node's record keys, reads back, refuses an adopted tunnel it does not name, and neither it nor the -// placement file names a field like a secret. +// Issue 1126 (ADR 0555): a node's record keys, reads back, refuses a node that is not a name and an empty capability, and +// neither it nor the placement file names a field like a secret. func TestNodeRecord(t *testing.T) { key, err := NodeRecordKey("ace") if err != nil || key != "record.ace" || NodeRecordsBucket != "node-engine_records" { @@ -31,9 +31,7 @@ func TestNodeRecord(t *testing.T) { hub := NodeRecord{Node: "novox", TunnelKey: "k", Capabilities: []string{"docker"}, SealingKey: "age1x", ServingKey: "pub", Certificate: "cert", OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, - NetworksSaid: true, Contract: 3, - Tunnel: &RecordTunnel{Interface: "wg0", PublicKey: "k", Peers: []RecordTunnelPeer{{PublicKey: "p"}}}, - HubAdopted: true, Carried: []RecordCarriedPeer{{PublicKey: "p", EnrolledAs: "phone"}}} + NetworksSaid: true, Contract: 3} raw, err := EncodeNodeRecord(hub) if err != nil { t.Fatal(err) @@ -44,7 +42,6 @@ func TestNodeRecord(t *testing.T) { } for name, r := range map[string]NodeRecord{ - "adopted with no tunnel": {Node: "novox", HubAdopted: true}, "empty capability": {Node: "ace", Capabilities: []string{""}}, "node that is not a name": {Node: "a.b"}, } { diff --git a/go/delivery/placement.go b/go/delivery/placement.go index 4f48a1b..293fb0e 100644 --- a/go/delivery/placement.go +++ b/go/delivery/placement.go @@ -119,7 +119,8 @@ type Placement struct { // lives in the catalogue, what the machine observes lives in its record, NodeRecord): where it is dialled (endpoint) and // its site, whether it is the private network's hub, its address on the private network, the domain its routed names // are composed under, its operator and agent logins by name and their homes, whether it is adopted rather than -// converged, and the modules taken on an adopted node. The address is a decision written once, at enrolment, by a +// converged, the modules taken on an adopted node, the tunnel found on it at adoption and, on the hub, the names given +// to that tunnel's peers. The address is a decision written once, at enrolment, by a // catalogue pull request: nothing allocates it at run time, and CheckPlacements refuses two nodes with one address. // No secret and no key material: accounts are names, never credentials. type NodeDecided struct { @@ -134,6 +135,10 @@ type NodeDecided struct { AgentHome string `json:"agent-account-home,omitempty"` Adopted bool `json:"adopted,omitempty"` Taken []string `json:"taken,omitempty"` + // Tunnel is the tunnel found on the node when it was adopted, kept as decided then: once the node takes it over it is + // no longer there to observe. PeerNames is, on the hub, the name a person gave a peer of that tunnel, by its public key. + Tunnel *RecordTunnel `json:"tunnel,omitempty"` + PeerNames map[string]string `json:"peer-names,omitempty"` } func (p Placement) check() error { From 2b0689f6546fca798593bbe6bd7479c05693758e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:29:45 +0100 Subject: [PATCH 08/42] =?UTF-8?q?A=20node's=20record=20rides=20in=20its=20?= =?UTF-8?q?own=20have=20key,=20.node:=20no=20bucket,=20writer=20or?= =?UTF-8?q?=20grant=20of=20its=20own,=20and=20its=20key's=20change=20is=20?= =?UTF-8?q?what=20readers=20hear=20(issue=201126,=20ADR=200555=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/frombus/frombus.go | 5 ++-- go/delivery/delivery.go | 11 +++++++-- go/delivery/noderecord.go | 43 +++++----------------------------- go/delivery/noderecord_test.go | 27 ++++++++++----------- 4 files changed, 31 insertions(+), 55 deletions(-) diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 94aa253..033bbcb 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -12,7 +12,7 @@ // - what a person decided of each node (its endpoint, site, hub, address on the private network, public domain and // accounts): the same placement key, Placement.Decided; // - what each node observes of itself (its tunnel key, capabilities, keys, certificate, links and networks): the -// node-engine's state `records`; the tunnel found at a node's adoption and the names of the hub's peers are decided, +// node's own have key, `.node` (HaveNode.Record); the tunnel found at a node's adoption and the names of the hub's peers are decided, // in its placement, and whether the hub took the tunnel over and who enrolled under a peer's key are read from both; // - every module's manifest: the indexer's `modules.` and its manifests by digest, or the digest the // composing node's want names where it runs another build (Scope.Kept); @@ -44,7 +44,8 @@ import ( type Reader interface { // Placements is every node's placement, by node (delivery.PlacementBucket). Placements(ctx context.Context) (map[string]delivery.Placement, error) - // Records is every node's record, by node (delivery.NodeRecordsBucket). + // Records is every node's record, by node: HaveNode.Record of each node's own have key, `.node` of + // delivery.HaveBucket (ADR 0555 §3). Records(ctx context.Context) (map[string]delivery.NodeRecord, error) // Modules is every module the indexer holds, by module (delivery.ModulesBucket). Modules(ctx context.Context) (map[string]delivery.Module, error) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 5eecc31..1c7834b 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -213,12 +213,15 @@ func parseModuleKeyPart(part string) (string, error) { return module, nil } +// HaveNodeWord is the last token of a node's own facts' key in HaveBucket. +const HaveNodeWord = nodeWord + // HaveNodeKey is the key, in HaveBucket, of a node's own facts: `.node`. func HaveNodeKey(node string) (string, error) { if err := checkName("node", node); err != nil { return "", err } - return node + ".node", nil + return node + "." + HaveNodeWord, nil } // ParseHaveKey is the inverse of HaveKey and HaveNodeKey: isNode is true for `.node`, with module empty. @@ -381,6 +384,9 @@ type HaveNode struct { Strays json.RawMessage `json:"strays,omitempty"` Filters json.RawMessage `json:"filters,omitempty"` Profile json.RawMessage `json:"profile,omitempty"` + // Record is what the machine observes of itself, typed, for every node and composing part to read (issue 1126, + // ADR 0555 §3): the node's record lives in its own have key, so it needs no bucket, writer or grant of its own. + Record *NodeRecord `json:"record,omitempty"` } // Presence is whether a node or a part's instance is online; the entry's own time says since when. The watchman @@ -524,7 +530,8 @@ func SameHave(a, b Have) bool { // SameHaveNode says whether two node facts are the same value, comparing each carried JSON without its spacing. func SameHaveNode(a, b HaveNode) bool { return a.V == b.V && a.Composed == b.Composed && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) && - sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) + sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) && + (a.Record == nil) == (b.Record == nil) && (a.Record == nil || SameNodeRecord(*a.Record, *b.Record)) } // SamePresence says whether two presences are the same value. diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go index 3b662b0..d7fef79 100644 --- a/go/delivery/noderecord.go +++ b/go/delivery/noderecord.go @@ -3,47 +3,16 @@ package delivery import ( "encoding/json" "fmt" - "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) -// A node's record is what the node knows of itself, written by its own agent into the node-engine's state `records`, -// one key `record.` per node, and read by every node and by the parts that compose (novox/hq ADR 0547 §2: "of -// other nodes it reads only what is public: their records"; issue 1126's prepare, L1 as narrowed by ADR 0549). Where a -// module runs, its host ports, pins, bindings and seat holdings are not in it: they are the catalogue's placement -// (placement.go). Nothing in it is secret: its keys are public halves, its accounts are login names. -// -// The agent writes it at start and whenever a fact it holds changes, only when the value differs, and says -// `node-record-changed`. Seeded from what the node runs and reports, never from the controller's store. -const ( - // NodeRecordsState is the node-engine's state of node records, and NodeRecordsBucket its bucket. - NodeRecordsState = "records" - NodeRecordsBucket = "node-engine_" + NodeRecordsState - // NodeRecordsHistory is the revisions the records bucket keeps per key. - NodeRecordsHistory = 5 - // NodeRecordChangedEvent is what a node's agent says when it wrote its record. - NodeRecordChangedEvent = "node-record-changed" - - recordWord = "record" -) - -// NodeRecordKey is the key, in NodeRecordsBucket, of one node's record: `record.`. -func NodeRecordKey(node string) (string, error) { - if err := checkName("node", node); err != nil { - return "", err - } - return recordWord + "." + node, nil -} - -// ParseNodeRecordKey is the inverse of NodeRecordKey; it refuses any other key, naming it. -func ParseNodeRecordKey(key string) (node string, err error) { - node, ok := strings.CutPrefix(key, recordWord+".") - if !ok || checkName("node", node) != nil { - return "", fmt.Errorf("delivery: %q is not a node record key, record.", key) - } - return node, nil -} +// A node's record is what the machine observes of itself (issue 1126, ADR 0555 §3), written by its own agent in its +// own have key, `.node` of the node-engine's have bucket (HaveNode.Record), whenever an observed fact changes and +// only when the value differs. The key's change is the event its readers hear: every node and the parts that compose +// watch it. Where a module runs, and every fact a person decides of the node (its endpoint, site, hub, address on the +// private network, public domain, accounts, the tunnel found at adoption), are the catalogue's placement +// (placement.go). Nothing in it is secret: its keys are public halves. // RecordTunnel is a tunnel found on a node when it was adopted, as compose.Tunnel holds it: public keys and addresses // only. After the node took it over the tunnel is no longer there to observe, so it is kept as what was decided at diff --git a/go/delivery/noderecord_test.go b/go/delivery/noderecord_test.go index e69e24f..e49b327 100644 --- a/go/delivery/noderecord_test.go +++ b/go/delivery/noderecord_test.go @@ -13,22 +13,9 @@ import ( // check (ADR 0201 §13) refuses such a field in a state every node reads. var secretNamed = regexp.MustCompile(`(?i)(secret|passw|token|api[-_]?key|private[-_]?key|credential|bearer|cookie|salt|signing)`) -// Issue 1126 (ADR 0555): a node's record keys, reads back, refuses a node that is not a name and an empty capability, and +// Issue 1126 (ADR 0555): a node's record reads back, rides in its have key, refuses a node that is not a name and an empty capability, and // neither it nor the placement file names a field like a secret. func TestNodeRecord(t *testing.T) { - key, err := NodeRecordKey("ace") - if err != nil || key != "record.ace" || NodeRecordsBucket != "node-engine_records" { - t.Fatalf("NodeRecordKey = %q, %v; bucket %q", key, err, NodeRecordsBucket) - } - if n, err := ParseNodeRecordKey(key); err != nil || n != "ace" { - t.Fatalf("ParseNodeRecordKey = %q, %v", n, err) - } - for _, bad := range []string{"placement.ace", "record.", "record.a.b"} { - if _, err := ParseNodeRecordKey(bad); err == nil { - t.Errorf("ParseNodeRecordKey(%q) took it", bad) - } - } - hub := NodeRecord{Node: "novox", TunnelKey: "k", Capabilities: []string{"docker"}, SealingKey: "age1x", ServingKey: "pub", Certificate: "cert", OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, NetworksSaid: true, Contract: 3} @@ -40,6 +27,18 @@ func TestNodeRecord(t *testing.T) { if err != nil || !SameNodeRecord(back, hub) || back.V != Version { t.Fatalf("round trip: %+v, %v", back, err) } + // It rides in the node's own have key: a have that differs only in its record is another value. + with, without := HaveNode{V: Version, Record: &hub}, HaveNode{V: Version} + if raw, err := EncodeHaveNode(with); err != nil { + t.Fatal(err) + } else if got, err := DecodeHaveNode(raw); err != nil || got.Record == nil || !SameHaveNode(got, with) || SameHaveNode(got, without) { + t.Fatalf("the have's record: %+v, %v", got, err) + } + changed := hub + changed.Contract = 4 + if SameHaveNode(with, HaveNode{V: Version, Record: &changed}) { + t.Error("a changed record reads as the same have") + } for name, r := range map[string]NodeRecord{ "empty capability": {Node: "ace", Capabilities: []string{""}}, From 9789c8379a7411f57b1f65e9ddb0632881f883dd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:33:36 +0100 Subject: [PATCH 09/42] =?UTF-8?q?Node=20principals=20read=20every=20node's?= =?UTF-8?q?=20own=20facts,=20where=20its=20record=20rides;=20the=20compose?= =?UTF-8?q?r=20copy=20is=20checked=20to=20grant=20all=20the=20controller's?= =?UTF-8?q?=20grants=20and=20only=20that=20and=20the=20issue=201790=20rows?= =?UTF-8?q?=20more=20(issue=201126,=20ADR=200555=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buscompose/buscompose_test.go | 32 ++++++++++++++++++++++++++++---- go/buscompose/nats.go | 4 ++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 1d361ca..60297ec 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -14,10 +14,26 @@ import ( // byte what the controller's composes. testdata/controller-user-lists.json was written by the controller's own // internal/broker (mesh-controller branch fix/1945-mesh-grants-publishes-memberships, and fix/1790-grants-create-buckets for the grants part creating declared streams), lists and memberships, from fixtures_test.go compiled there unchanged; composed.conf is the // controller's golden of the whole server configuration. A row that differs is the copy drifting from the controller, -// which is a fault while both compose (until the controller loses its lease, ADR 0547 §13). +// which is a fault while both compose (until the controller loses its lease, ADR 0547 §13), but for the rows the copy +// grants beyond it on purpose: the copy must grant everything the controller's grants, and only those rows more. // createdAdvisory is one subscription to a bucket's creation in a composed list, as stateCreatedHeard composes it. var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATED\.KV_[^"]*", `) +// nodeRecordReads are the two rows the copy grants a node principal beyond the controller's on purpose (novox/hq ADR 0555 +// §3, issue 1126): a read of every node's own facts, `.node` of the have bucket, where the node's record rides, so a +// node agent composing its own modules reads the other nodes' public records. After the take-over (ADR 0547 §8) the grants +// part's list is the one the bus loads; a hand-back loads the controller's, which lacks these reads, and keeps working +// without the second milestone's composing. +var nodeRecordReads = regexp.MustCompile(`"\$JS\.API\.(DIRECT\.GET|CONSUMER\.CREATE)\.KV_node-engine_have\.(\*\.)?\$KV\.node-engine_have\.\*\.node",? ?`) + +// beyondTheControllers is a list the copy composed with the rows it grants beyond the controller's on purpose taken +// out, and how many node-record rows it held: what is left must be the controller's list exactly, so the copy grants +// everything the controller's grants and nothing more but those rows. +func beyondTheControllers(list string) (string, int) { + n := len(nodeRecordReads.FindAllString(list, -1)) + return nodeRecordReads.ReplaceAllString(createdAdvisory.ReplaceAllString(list, ""), ""), n +} + func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { raw, err := os.ReadFile(filepath.Join("testdata", "controller-user-lists.json")) if err != nil { @@ -56,9 +72,13 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { // the take-over of ADR 0547 milestone 2): each principal hears its own and read buckets created, so a state // missing at its start is bound when it appears. The controller's composer is frozen and its list no longer // loaded; while it is not, this row is left out of the comparison and nothing else. - list = createdAdvisory.ReplaceAllString(list, "") + list, records := beyondTheControllers(list) if list != want[i].List { - t.Errorf("the list differs from the controller's:\n%s", list) + t.Errorf("the list differs from the controller's beyond the intended rows:\n%s", list) + } + // And the intended rows are there: two for each node principal of the fixture. + if nodes := strings.Count(want[i].List, `"$KV.node-engine_have.`); nodes > 0 && records == 0 { + t.Errorf("no node principal reads the other nodes' records") } // And every membership the records issue, as the controller's MembershipFor makes it (ADR 0547 §5): the // grants part publishes them. @@ -135,8 +155,12 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { if err != nil { t.Fatal(err) } + got, records := beyondTheControllers(got) + if records != 2 { + t.Errorf("node one is granted %d of the two reads of the other nodes' records", records) + } if got != string(want) { - t.Errorf("the configuration differs from the controller's golden:\n%s", got) + t.Errorf("the configuration differs from the controller's golden beyond the intended rows:\n%s", got) } }) } diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 1ff25c8..cdf7337 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -882,6 +882,10 @@ func PermissionsFor(p Principal) (Permissions, error) { return Permissions{}, err } pub = append(pub, haveKeys...) + // And every node's own facts, `.node`, its record among them (issue 1126, ADR 0555 §3): a node agent composing + // its own modules reads the other nodes' public records, which its firewall and resolver need (ADR 0547 §2). Read + // only: each node still writes its own keys alone. + pub = append(pub, readGrants(delivery.HaveBucket, "*."+delivery.HaveNodeWord)...) // And an object of the declaration bucket by its name, and nothing of the bucket's list or writing // (novox/hq ADR 0486 §2, issue 1697). fileReads, err := DeclarationFilesReads(p.Node) From e49309d61c0d27c345e2ccd84f9f2290ca96bae4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:39:46 +0100 Subject: [PATCH 10/42] =?UTF-8?q?A=20want=20names=20the=20build=20alone=20?= =?UTF-8?q?from=20the=20second=20milestone,=20and=20the=20yes=20a=20person?= =?UTF-8?q?=20gave;=20a=20have=20waits=20for=20that=20yes=20with=20its=20j?= =?UTF-8?q?udgement=20(issue=201126,=20ADR=200547=20=C2=A72,=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/assigner.go | 23 +++++++++++++++++------ go/delivery/delivery.go | 20 +++++++++++++++++++- go/delivery/delivery_test.go | 17 +++++++++++++++++ go/delivery/want_composed_test.go | 12 ++++++++---- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/go/delivery/assigner.go b/go/delivery/assigner.go index 4aff2fd..7bb559e 100644 --- a/go/delivery/assigner.go +++ b/go/delivery/assigner.go @@ -166,8 +166,9 @@ func checkFingerprint(fp, of string) error { return nil } -// Want is a node's want for a module: the fingerprint FP the node should run and the digest Composed of the one -// composition of it the node may apply (ADR 0539 clause 1), or Removed for a module that left the node's modules (then +// Want is a node's want for a module: the fingerprint FP the node should run and, before ADR 0547's second milestone, +// the digest Composed of the one composition of it the node may apply (ADR 0539 clause 1; from that milestone the +// node composes its own, ADR 0547 §2, and a want names none), or Removed for a module that left the node's modules (then // both are empty). When the module's newest fingerprint is not yet allowed on this node, Newest names it and Rule the // rule that holds the node back; both are empty when FP is the newest. Backup is the answer to a failed backup before // the apply, BackupAgain or BackupSkip (clause 16), empty when none was asked. Words say why in a sentence. The @@ -185,6 +186,10 @@ type Want struct { // is released. A put's want names no newest fingerprint or rule. Put bool `json:"put,omitempty"` Hold bool `json:"hold,omitempty"` + // Yes is the digest of a composition a person allowed the node to move to (novox/hq ADR 0547 §3): a move of a module + // of policy record that the node agent judged to recreate a container or move its data is applied only when its + // want names the composition so allowed. Empty when none was asked. + Yes string `json:"yes,omitempty"` } func (w Want) check() error { @@ -203,11 +208,17 @@ func (w Want) check() error { if err := checkFingerprint(w.FP, "a want's fingerprint"); err != nil { return err } - if w.Composed == "" { - return fmt.Errorf("delivery: the want of %s names no composition: a want names the digest of the one composition it allows (ADR 0539 clause 1)", w.FP) + // From ADR 0547's second milestone the node composes its own modules (§2), and a want names the build alone: no + // composition digest. A want written before names one, which is read as it was written. + if w.Composed != "" { + if err := checkDigest(w.Composed, "a want's composition digest"); err != nil { + return err + } } - if err := checkDigest(w.Composed, "a want's composition digest"); err != nil { - return err + if w.Yes != "" { + if err := checkDigest(w.Yes, "the digest a want's yes allows"); err != nil { + return err + } } switch w.Backup { case "", BackupAgain, BackupSkip: diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 1c7834b..1966d1b 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -97,6 +97,10 @@ const ( // composition is applied, and the node waits for the answer in its want (novox/hq ADR 0539 clause 16). Detail // carries the backup's own words. PhaseHeld Phase = "held" + // PhaseWaitingYes is a module of policy record whose composed move the node agent judged to recreate a container or + // move its data (novox/hq ADR 0547 §3): nothing of it is applied, Judgement says the judgement and Waits the digest + // of the composition waiting, and the assigner asks a person; the node applies it once its want's Yes names Waits. + PhaseWaitingYes Phase = "waiting-yes" ) // Health is what the module's own check says. @@ -340,6 +344,10 @@ type Have struct { // UserList is what the node-engine of the node holding mesh-broker wrote of the bus's user list, in the have of the // bus module (novox/hq ADR 0547 §7, §8). Nil on every other have. UserList *UserListWritten `json:"user-list,omitempty"` + // Judgement and Waits are, in phase waiting-yes, the node agent's judgement of the move and the digest of the + // composition that waits for a person's yes (ADR 0547 §3); empty otherwise. + Judgement Judgement `json:"judgement,omitempty"` + Waits string `json:"waits,omitempty"` } // UserListWritten is the bus's user list as the node-engine of the bus node wrote it to the bus module's file, applied @@ -435,8 +443,18 @@ func decode(data []byte, into any) error { func (h Have) check() error { switch h.Phase { case PhaseBackingUp, PhaseApplying, PhaseChecking, PhaseRunning, PhaseFailed, PhaseRemoved, PhaseHeld: + if h.Judgement != "" || h.Waits != "" { + return fmt.Errorf("delivery: only a have waiting for a yes names a judgement and the composition waiting") + } + case PhaseWaitingYes: + if h.Judgement != JudgedRecreates && h.Judgement != JudgedMovesData && h.Judgement != JudgedCannot { + return fmt.Errorf("delivery: a have waiting for a yes names the judgement %q, not recreates, moves-data or cannot-judge", h.Judgement) + } + if err := checkDigest(h.Waits, "the composition a have waits a yes for"); err != nil { + return err + } default: - return fmt.Errorf("delivery: phase %q is not backing-up, applying, checking, running, failed, removed or held", h.Phase) + return fmt.Errorf("delivery: phase %q is not backing-up, applying, checking, running, failed, removed, held or waiting-yes", h.Phase) } if h.Composed != "" { if err := checkDigest(h.Composed, "the have's composition digest"); err != nil { diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index 9e6b68c..2f3c480 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -367,6 +367,23 @@ func testPhaseHealthRefused(t *testing.T) { if _, err := DecodeHave([]byte(`{"v":1,"phase":"running","health":"fine"}`)); err == nil { t.Error("bad health decoded") } + // ADR 0547 §3 (issue 1126): a have waits for a yes with its judgement and the composition waiting, and no other have + // names either. + waits := Have{Phase: PhaseWaitingYes, Health: HealthOK, Judgement: JudgedRecreates, Waits: "sha256:" + strings.Repeat("a", 64)} + if raw, err := EncodeHave(waits); err != nil { + t.Errorf("a have waiting for a yes: %v", err) + } else if back, err := DecodeHave(raw); err != nil || back.Judgement != JudgedRecreates || back.Waits != waits.Waits { + t.Errorf("a have waiting for a yes came back %+v, %v", back, err) + } + for name, h := range map[string]Have{ + "neither judged": {Phase: PhaseWaitingYes, Health: HealthOK, Judgement: JudgedNeither, Waits: waits.Waits}, + "no waiting digest": {Phase: PhaseWaitingYes, Health: HealthOK, Judgement: JudgedMovesData}, + "running judged": {Phase: PhaseRunning, Health: HealthOK, Judgement: JudgedRecreates}, + } { + if _, err := EncodeHave(h); err == nil { + t.Errorf("%s: encoded", name) + } + } } // 1131/AC7: the pulse's time to live is 120 seconds minus the move's age; at the bound the move is stalled. diff --git a/go/delivery/want_composed_test.go b/go/delivery/want_composed_test.go index 12512e8..7c9d68d 100644 --- a/go/delivery/want_composed_test.go +++ b/go/delivery/want_composed_test.go @@ -8,7 +8,8 @@ import ( func hashOf(c byte) string { return "sha256:" + digestOf(c) } // 1943/AC3: a want naming its fingerprint and the digest of the composition it allows round-trips, and so does one -// carrying the answer to a failed backup; a want naming a fingerprint without a digest, a digest not of the sha256 +// carrying the answer to a failed backup, and from ADR 0547's second milestone one naming the build alone or the yes +// a person gave; a digest not of the sha256 // form, a removed want naming a digest, an unknown backup answer and a rule past the seventh are refused. func TestAWantNamesItsComposition(t *testing.T) { for _, w := range []Want{ @@ -17,6 +18,9 @@ func TestAWantNamesItsComposition(t *testing.T) { {FP: fp2of('a'), Composed: hashOf('1'), Backup: BackupSkip, Words: "the operator: go on without a backup"}, {FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: RuleComposed, Words: "no composition of b for ace yet"}, {Removed: true}, + // ADR 0547 §2, §3 (issue 1126): from the second milestone a want names the build alone, and the yes a person gave. + {FP: fp2of('a')}, + {FP: fp2of('a'), Yes: hashOf('3')}, } { raw, err := EncodeWant(w) if err != nil { @@ -32,7 +36,7 @@ func TestAWantNamesItsComposition(t *testing.T) { w Want why string }{ - {Want{FP: fp2of('a')}, "names no composition"}, + {Want{FP: fp2of('a'), Yes: "sha256:12"}, "a want's yes allows"}, {Want{FP: fp2of('a'), Composed: "sha256:12"}, "not sha256"}, {Want{Removed: true, Composed: hashOf('1')}, "removed want names no"}, {Want{FP: fp2of('a'), Composed: hashOf('1'), Backup: "later"}, `backup answer "later"`}, @@ -44,8 +48,8 @@ func TestAWantNamesItsComposition(t *testing.T) { t.Errorf("EncodeWant(%+v) = %v, want a refusal saying %q", r.w, err, r.why) } } - if _, err := DecodeWant([]byte(`{"v":1,"fp":"` + fp2of('a') + `"}`)); err == nil || !strings.Contains(err.Error(), "names no composition") { - t.Errorf("a want of release 5, naming no composition, was read: %v", err) + if w, err := DecodeWant([]byte(`{"v":1,"fp":"` + fp2of('a') + `"}`)); err != nil || w.Composed != "" { + t.Errorf("a want naming the build alone (ADR 0547 §2) is not read: %+v, %v", w, err) } if RuleComposed != 7 { t.Errorf("RuleComposed = %d, want the seventh rule", RuleComposed) From 9532b7872420d3ccb5c41551ab1746dffb44a5c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:41:33 +0100 Subject: [PATCH 11/42] =?UTF-8?q?A=20node's=20own=20want=20names=20no=20co?= =?UTF-8?q?mposition=20from=20the=20second=20milestone:=20the=20node=20com?= =?UTF-8?q?poses=20its=20own=20(issue=201126,=20ADR=200547=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/assigner.go | 15 ++++++++++----- go/delivery/want_composed_test.go | 8 +++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/go/delivery/assigner.go b/go/delivery/assigner.go index 7bb559e..6c5d8e2 100644 --- a/go/delivery/assigner.go +++ b/go/delivery/assigner.go @@ -322,10 +322,13 @@ type NodeWant struct { Words string `json:"words,omitempty"` } -// EncodeNodeWant is w as JSON at Version; it refuses a digest not of the sha256 form. +// EncodeNodeWant is w as JSON at Version; it refuses a digest not of the sha256 form. From ADR 0547's second milestone +// the node composes its own composition (§2) and its own want names none: an empty Composed is that want. func EncodeNodeWant(w NodeWant) ([]byte, error) { - if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil { - return nil, err + if w.Composed != "" { + if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil { + return nil, err + } } w.V = Version return json.Marshal(w) @@ -343,8 +346,10 @@ func DecodeNodeWant(data []byte) (NodeWant, error) { if err := d.Decode(&w); err != nil { return NodeWant{}, fmt.Errorf("delivery: a node's own want does not fit its shape: %w", err) } - if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil { - return NodeWant{}, err + if w.Composed != "" { + if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil { + return NodeWant{}, err + } } return w, nil } diff --git a/go/delivery/want_composed_test.go b/go/delivery/want_composed_test.go index 7c9d68d..98d3476 100644 --- a/go/delivery/want_composed_test.go +++ b/go/delivery/want_composed_test.go @@ -90,11 +90,17 @@ func TestTheNodesOwnWant(t *testing.T) { if err != nil || back != w { t.Errorf("node want came back %+v, %v", back, err) } - for _, bad := range []NodeWant{{}, {Composed: "sha256:1"}} { + for _, bad := range []NodeWant{{Composed: "sha256:1"}} { if _, err := EncodeNodeWant(bad); err == nil { t.Errorf("EncodeNodeWant(%+v) accepted", bad) } } + // ADR 0547 §2 (issue 1126): from the second milestone the node composes its own composition, and its want names none. + if raw, err := EncodeNodeWant(NodeWant{Words: "the node composes its own"}); err != nil { + t.Errorf("a node want naming no composition: %v", err) + } else if back, err := DecodeNodeWant(raw); err != nil || back.Composed != "" { + t.Errorf("a node want naming no composition came back %+v, %v", back, err) + } if _, err := DecodeNodeWant([]byte(`{"v":1,"composed":"` + hashOf('2') + `","fp":"` + fp2of('a') + `"}`)); err == nil { t.Errorf("a node want naming a fingerprint was read") } From 6df0dfe208c666be56f73ea897490c52c60fe35a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:44:06 +0100 Subject: [PATCH 12/42] =?UTF-8?q?Node=20principals=20read=20what=20they=20?= =?UTF-8?q?compose=20their=20own=20modules=20from;=20the=20composer=20copy?= =?UTF-8?q?=20is=20held=20per=20user=20to=20the=20controller's=20grants=20?= =?UTF-8?q?plus=20exactly=20these=20(issue=201126,=20ADR=200547=20=C2=A72,?= =?UTF-8?q?=20ADR=200549=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buscompose/buscompose_test.go | 117 +++++++++++++++++++++++-------- go/buscompose/nats.go | 4 ++ go/buscompose/state.go | 16 +++++ 3 files changed, 108 insertions(+), 29 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 60297ec..554c6a1 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -14,24 +14,96 @@ import ( // byte what the controller's composes. testdata/controller-user-lists.json was written by the controller's own // internal/broker (mesh-controller branch fix/1945-mesh-grants-publishes-memberships, and fix/1790-grants-create-buckets for the grants part creating declared streams), lists and memberships, from fixtures_test.go compiled there unchanged; composed.conf is the // controller's golden of the whole server configuration. A row that differs is the copy drifting from the controller, -// which is a fault while both compose (until the controller loses its lease, ADR 0547 §13), but for the rows the copy -// grants beyond it on purpose: the copy must grant everything the controller's grants, and only those rows more. +// which is a fault while both compose (until the controller loses its lease, ADR 0547 §13), but for the grants the copy +// gives beyond it on purpose (beyondTheControllers): the copy grants every user everything the controller's grants, and +// only those more. // createdAdvisory is one subscription to a bucket's creation in a composed list, as stateCreatedHeard composes it. var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATED\.KV_[^"]*", `) -// nodeRecordReads are the two rows the copy grants a node principal beyond the controller's on purpose (novox/hq ADR 0555 -// §3, issue 1126): a read of every node's own facts, `.node` of the have bucket, where the node's record rides, so a -// node agent composing its own modules reads the other nodes' public records. After the take-over (ADR 0547 §8) the grants -// part's list is the one the bus loads; a hand-back loads the controller's, which lacks these reads, and keeps working -// without the second milestone's composing. -var nodeRecordReads = regexp.MustCompile(`"\$JS\.API\.(DIRECT\.GET|CONSUMER\.CREATE)\.KV_node-engine_have\.(\*\.)?\$KV\.node-engine_have\.\*\.node",? ?`) +// beyondTheControllers is what the copy grants beyond the controller's on purpose (novox/hq ADR 0547 §2, ADR 0555 §3, +// issue 1126), by user: a node's account reads every node's own facts, `.node` of the have bucket where the node's +// record rides, and what it composes its own modules from (NodeComposeReads). After the take-over (ADR 0547 §8) the +// grants part's list is the one the bus loads; a hand-back loads the controller's, which lacks these reads, and the nodes +// keep applying without composing their own until the next take-over. Each principal's hearing of its buckets' creation +// (issue 1790) is the other row beyond, matched by createdAdvisory. +func beyondTheControllers(user string) map[string]bool { + out := map[string]bool{} + node, ok := strings.CutPrefix(user, "node.") + if !ok { + return out + } + for _, s := range append(readGrants("node-engine_have", "*.node"), NodeComposeReads(node)...) { + out[s] = true + } + return out +} -// beyondTheControllers is a list the copy composed with the rows it grants beyond the controller's on purpose taken -// out, and how many node-record rows it held: what is left must be the controller's list exactly, so the copy grants -// everything the controller's grants and nothing more but those rows. -func beyondTheControllers(list string) (string, int) { - n := len(nodeRecordReads.FindAllString(list, -1)) - return nodeRecordReads.ReplaceAllString(createdAdvisory.ReplaceAllString(list, ""), ""), n +// quotedToken is a quoted token of a composed list. +var quotedToken = regexp.MustCompile(`"[^"]*"`) + +// grantsByUser splits a composed list into the text outside its users and each user's quoted tokens (its name, its +// password's hash, every subject it may publish or subscribe to), by user. +func grantsByUser(list string) (string, map[string]map[string]bool) { + var outside strings.Builder + users := map[string]map[string]bool{} + user := "" + for _, line := range strings.SplitAfter(list, "\n") { + trimmed := strings.TrimSpace(line) + if name, ok := strings.CutPrefix(trimmed, `{ user: "`); ok { + user, _, _ = strings.Cut(name, `"`) + users[user] = map[string]bool{} + } else if user != "" && trimmed == "]" { + user = "" + } + if user == "" { + outside.WriteString(line) + continue + } + for _, q := range quotedToken.FindAllString(line, -1) { + users[user][strings.Trim(q, `"`)] = true + } + } + return outside.String(), users +} + +// grantsAsTheControllers fails t unless the copy's list is the controller's, user for user, but for the grants it +// names on purpose (beyondTheControllers, the creation advisories): every grant of the controller's is the copy's, and +// the copy's beyond them are exactly those, every one of them there. +func grantsAsTheControllers(t *testing.T, got, want string) { + t.Helper() + gotOutside, gotUsers := grantsByUser(got) + wantOutside, wantUsers := grantsByUser(want) + if gotOutside != wantOutside { + t.Errorf("the list differs from the controller's outside its users:\n%s", gotOutside) + } + for user, was := range wantUsers { + now, ok := gotUsers[user] + if !ok { + t.Errorf("the copy has no user %s", user) + continue + } + intended := beyondTheControllers(user) + for g := range was { + if !now[g] { + t.Errorf("%s: the copy does not grant %s, which the controller's does", user, g) + } + } + for g := range now { + if !was[g] && !intended[g] && !createdAdvisory.MatchString(`"`+g+`", `) { + t.Errorf("%s: the copy grants %s beyond the controller's, and no decision names it", user, g) + } + } + for g := range intended { + if !now[g] { + t.Errorf("%s: the copy does not grant %s, which a node composing its own modules reads", user, g) + } + } + } + for user := range gotUsers { + if _, ok := wantUsers[user]; !ok { + t.Errorf("the copy has a user %s the controller's has not", user) + } + } } func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { @@ -72,14 +144,7 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { // the take-over of ADR 0547 milestone 2): each principal hears its own and read buckets created, so a state // missing at its start is bound when it appears. The controller's composer is frozen and its list no longer // loaded; while it is not, this row is left out of the comparison and nothing else. - list, records := beyondTheControllers(list) - if list != want[i].List { - t.Errorf("the list differs from the controller's beyond the intended rows:\n%s", list) - } - // And the intended rows are there: two for each node principal of the fixture. - if nodes := strings.Count(want[i].List, `"$KV.node-engine_have.`); nodes > 0 && records == 0 { - t.Errorf("no node principal reads the other nodes' records") - } + grantsAsTheControllers(t, list, want[i].List) // And every membership the records issue, as the controller's MembershipFor makes it (ADR 0547 §5): the // grants part publishes them. raw, err := json.Marshal(MembershipsFor(f)) @@ -155,12 +220,6 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { if err != nil { t.Fatal(err) } - got, records := beyondTheControllers(got) - if records != 2 { - t.Errorf("node one is granted %d of the two reads of the other nodes' records", records) - } - if got != string(want) { - t.Errorf("the configuration differs from the controller's golden beyond the intended rows:\n%s", got) - } + grantsAsTheControllers(t, got, string(want)) }) } diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index cdf7337..8f5379b 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -886,6 +886,10 @@ func PermissionsFor(p Principal) (Permissions, error) { // its own modules reads the other nodes' public records, which its firewall and resolver need (ADR 0547 §2). Read // only: each node still writes its own keys alone. pub = append(pub, readGrants(delivery.HaveBucket, "*."+delivery.HaveNodeWord)...) + // And what it composes its own modules from (ADR 0547 §2, ADR 0549 §3): the indexer's placement, modules and + // manifests, the builder's builds (the manifest of a build its want names), and mesh-config's resolved values of + // its own node alone. Read only. + pub = append(pub, NodeComposeReads(p.Node)...) // And an object of the declaration bucket by its name, and nothing of the bucket's list or writing // (novox/hq ADR 0486 §2, issue 1697). fileReads, err := DeclarationFilesReads(p.Node) diff --git a/go/buscompose/state.go b/go/buscompose/state.go index f772cae..a0d6feb 100644 --- a/go/buscompose/state.go +++ b/go/buscompose/state.go @@ -3,6 +3,7 @@ package buscompose import ( "fmt" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/delivery" "sort" "strings" "time" @@ -306,3 +307,18 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err sort.Strings(undeclared) return undeclared, nil } + +// ResolvedBucket is mesh-config's state of resolved values, keyed `..`. +const ResolvedBucket = "mesh-config_resolved" + +// NodeComposeReads is what a node's agent reads to compose its own modules (novox/hq ADR 0547 §2, ADR 0549 §3, issue +// 1126): every key of the indexer's placement, modules and manifests and of the builder's builds, and of mesh-config's +// resolved values only its own node's, `.>`. Reads only; the node is checked as every subject token is by the +// caller's principal. +func NodeComposeReads(node string) []string { + var out []string + for _, bucket := range []string{delivery.PlacementBucket, delivery.ModulesBucket, delivery.ManifestsBucket, delivery.BuilderBuildsBucket} { + out = append(out, readGrants(bucket, "")...) + } + return append(out, readGrants(ResolvedBucket, node+".>")...) +} From b72de2b75a725e2ef0d35191b38a358566d59836 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:59:05 +0100 Subject: [PATCH 13/42] =?UTF-8?q?A=20node=20applying=20what=20it=20compose?= =?UTF-8?q?d=20refuses=20a=20composition=20that=20stands=20in=20for=20anyt?= =?UTF-8?q?hing=20its=20input=20lacks,=20the=20bus's=20user=20list=20alone?= =?UTF-8?q?=20held=20out=20(issue=201126,=20ADR=200547=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/composed.go | 40 ++++++++++++++++++++++++++++++++++ go/compose/credentials_test.go | 11 ++++++++++ go/compose/frombus/frombus.go | 7 +++--- go/compose/render.go | 2 +- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/go/compose/composed.go b/go/compose/composed.go index 9fe66a3..651357d 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -2,8 +2,10 @@ package compose import ( "encoding/json" + "errors" "fmt" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -16,6 +18,26 @@ import ( // (mesh-hand-compose) gets that one. Each module's Data says whether its manifest declares data. The compositions are not // signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest). func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) { + return compositions(in, node, fps, mode, false) +} + +// ErrStoodIn is a composition that holds a stand-in for a value the input does not carry (an own secret, a grant's +// sealed copy, a vault box, the certificate, the bus's user list) or a port it does not give: what a send would make, +// never what may be applied. +var ErrStoodIn = errors.New("compose: the composition stands in for what its input does not carry") + +// BusUsersStandIn is what a composition says it stood in for when its input carries no bus user list. +const BusUsersStandIn = "the bus's user list" + +// CompositionsApplied is Compositions for a node that applies what it composed itself (novox/hq ADR 0547 §2, issue +// 1126): a declaration that stands in for anything (Declared.StoodIn, Foreseen, ForeseenPorts) is refused with +// ErrStoodIn, naming each, so a stand-in is never applied as if it were the value; but for the bus's user list +// (BusUsersStandIn), which the bus node holds out of every apply of a composition it made itself. +func CompositionsApplied(in Input, node string, fps map[string]string) (map[string]delivery.Composed, []string, error) { + return compositions(in, node, fps, Reading, true) +} + +func compositions(in Input, node string, fps map[string]string, mode Mode, applied bool) (out map[string]delivery.Composed, skipped []string, err error) { reading := in reading.Nodes = append([]Node(nil), in.Nodes...) found := false @@ -35,6 +57,24 @@ func Compositions(in Input, node string, fps map[string]string, mode Mode) (out if len(problems) > 0 { return nil, nil, fmt.Errorf("compose: %s's declaration is refused by the node-engine's validator: %v", node, problems) } + if applied { + var stood []string + for _, x := range d.StoodIn { + // The bus's user list is the bus node's own to follow, written alone and never from a composition the node + // made itself (ADR 0547 §7, §8): its stand-in is held out of the apply, not applied. + if x != BusUsersStandIn { + stood = append(stood, x) + } + } + stood = append(stood, d.Foreseen...) + for _, p := range d.ForeseenPorts { + stood = append(stood, "port "+p) + } + if len(stood) > 0 { + sort.Strings(stood) + return nil, nil, fmt.Errorf("%w on %s: %s", ErrStoodIn, node, strings.Join(stood, ", ")) + } + } body, err := d.Body() if err != nil { return nil, nil, err diff --git a/go/compose/credentials_test.go b/go/compose/credentials_test.go index ba4db0c..924df06 100644 --- a/go/compose/credentials_test.go +++ b/go/compose/credentials_test.go @@ -2,6 +2,7 @@ package compose import ( "encoding/json" + "errors" "strings" "testing" @@ -78,6 +79,16 @@ func TestAnOwnSecretHeldIsCarriedAndOneNotHeldIsForeseenOrLeftOut(t *testing.T) t.Errorf("a held secret with no value is not named: %v", got.Rendering.Foreseen) } unchanged(t, mk, func(in Input) { rendered(t, in, "laptop", Foreseeing) }) + + // 1126 (ADR 0547 §2): a node applying what it composed itself never applies a stand-in: a held secret with no value in the + // Input is refused, naming it; held with its value, it composes. + fps := map[string]string{"assistant": "src2:" + strings.Repeat("a", 64), overlay.Name: "src2:" + strings.Repeat("b", 64)} + if _, _, err := CompositionsApplied(in, "laptop", fps); !errors.Is(err, ErrStoodIn) || !strings.Contains(err.Error(), "assistant/password") { + t.Errorf("a stand-in was not refused for a node's own apply: %v", err) + } + if out, _, err := CompositionsApplied(mk(), "laptop", fps); err != nil || len(out) == 0 { + t.Errorf("a held secret with its value is refused for a node's own apply: %v", err) + } } // An own secret issued outside the mesh that nobody gave is never made (novox/hq issue 401, ADR 0228): nothing is composed or diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 033bbcb..0c7386a 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -201,14 +201,15 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er return in, nil } -// Compose is the node's compositions over the bus (ADR 0547 §2): Input for s, then compose.Compositions in the Reading -// mode at the fingerprints fps gives each module, as the controller composes them. +// Compose is the node's compositions over the bus (ADR 0547 §2): Input for s, then compose.CompositionsApplied at the +// fingerprints fps gives each module: composed in the Reading mode as the controller composes them, and refused when it +// stands in for anything the input does not carry (compose.ErrStoodIn), since the node applies what it composed. func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]string) (map[string]delivery.Composed, []string, error) { in, err := Input(ctx, r, mesh, s) if err != nil { return nil, nil, err } - return compose.Compositions(in, s.Node, fps, compose.Reading) + return compose.CompositionsApplied(in, s.Node, fps) } // nodeOf is the composition's node from its record (what the machine observes) and its placement (what a person diff --git a/go/compose/render.go b/go/compose/render.go index a50a65e..e8dca30 100644 --- a/go/compose/render.go +++ b/go/compose/render.go @@ -113,7 +113,7 @@ func (c *composer) render(nodeName string, mode Mode) (Rendered, error) { r.BusUsers = c.in.BusUsers if r.BusUsers == "" { r.BusUsers = ForeseenBusUsers - foreseen("the bus's user list") + foreseen(BusUsersStandIn) } if len(c.in.BusMissing) > 0 { out.Said = append(out.Said, fmt.Sprintf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+ From 22bc4dc8ef4ffd4eea6f705753bd8f010804237f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 18:59:35 +0100 Subject: [PATCH 14/42] MergeInto names the changed files that are compiled programs, read in the workspace, so a check can refuse build output (issue 1577) --- go/buildstep/merge.go | 8 +++ go/buildstep/merge_integration_test.go | 9 ++-- go/buildstep/programs.go | 72 ++++++++++++++++++++++++++ go/buildstep/programs_test.go | 72 ++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 go/buildstep/programs.go create mode 100644 go/buildstep/programs_test.go diff --git a/go/buildstep/merge.go b/go/buildstep/merge.go index 907009e..589c50e 100644 --- a/go/buildstep/merge.go +++ b/go/buildstep/merge.go @@ -15,6 +15,9 @@ type MergeResult struct { Base, Head, Merge string Conflicts []string Changed []string + // Programs are the changed files that are compiled programs (ExecutableKind of their first bytes, read in the workspace + // at the merge); never nil for a clean merge, so a caller tells none from a module that reports none (issue 1577). + Programs []Program } // MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit @@ -98,6 +101,11 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err) } res.Changed = strings.Fields(changed) + raw, err := git("diff", "--raw", "-z", "--no-renames", "--diff-filter=AM", base, res.Merge) + if err != nil { + return res, fmt.Errorf("the files the merge adds or modifies cannot be read: %w", err) + } + res.Programs = programsIn(dir, raw) return res, nil } diff --git a/go/buildstep/merge_integration_test.go b/go/buildstep/merge_integration_test.go index 88b6d11..ce8518c 100644 --- a/go/buildstep/merge_integration_test.go +++ b/go/buildstep/merge_integration_test.go @@ -102,7 +102,7 @@ func (r *mergeRig) fetch(head string) { // holds both sides' changes. func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing.T) { r := newMergeRig(t) - head := r.head(map[string]string{"b.txt": "head\n"}) + head := r.head(map[string]string{"b.txt": "head\n", "prog": "\x7fELF\x02\x01"}) base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"}) r.fetch(head) res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote()) @@ -112,8 +112,11 @@ func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing. if res.Base != base || res.Head != head || len(res.Merge) != 40 || res.Merge == base || res.Merge == head || len(res.Conflicts) != 0 { t.Fatalf("result = %+v", res) } - if !reflect.DeepEqual(res.Changed, []string{"b.txt"}) { - t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt", res.Changed) + if !reflect.DeepEqual(res.Changed, []string{"b.txt", "prog"}) { + t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt, prog", res.Changed) + } + if !reflect.DeepEqual(res.Programs, []Program{{"prog", "ELF"}}) { // issue 1577 + t.Errorf("programs = %v, want the ELF the head adds", res.Programs) } if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != res.Merge { t.Errorf("the workspace is at %s, not at the merge %s", at, res.Merge) diff --git a/go/buildstep/programs.go b/go/buildstep/programs.go new file mode 100644 index 0000000..8d87dce --- /dev/null +++ b/go/buildstep/programs.go @@ -0,0 +1,72 @@ +package buildstep + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" +) + +// Program is a changed file whose first bytes are an executable binary: a compiled program, which is build output and not +// source (novox/hq issue 1577: `go build` in a module's folder writes one beside go.mod, and `git add -A` takes it). +type Program struct { + Path string `json:"path"` + Kind string `json:"kind"` // ELF, Mach-O or PE +} + +// ExecutableKind says what executable format the first bytes of a file are, or "" when they are none. A PE needs its header +// ("PE\0\0" at the offset in bytes 0x3c) so a text starting with "MZ" is not one; a fat Mach-O (0xCAFEBABE) is told from a +// Java class by its architecture count (bytes 4 to 7, fewer than 0x20; a class has its version there). +func ExecutableKind(head []byte) string { + switch { + case bytes.HasPrefix(head, []byte("\x7fELF")): + return "ELF" + case bytes.HasPrefix(head, []byte("MZ")): + if len(head) >= 0x40 { + off := int(head[0x3c]) | int(head[0x3d])<<8 | int(head[0x3e])<<16 | int(head[0x3f])<<24 + if off >= 0 && off+4 <= len(head) && bytes.Equal(head[off:off+4], []byte("PE\x00\x00")) { + return "PE" + } + } + case bytes.HasPrefix(head, []byte{0xcf, 0xfa, 0xed, 0xfe}), bytes.HasPrefix(head, []byte{0xce, 0xfa, 0xed, 0xfe}), + bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xcf}), bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xce}), + bytes.HasPrefix(head, []byte{0xca, 0xfe, 0xba, 0xbe}) && len(head) >= 8 && head[4] == 0 && head[5] == 0 && head[6] == 0 && head[7] < 0x20: + return "Mach-O" + } + return "" +} + +// programHeadSize is how many bytes of a file are read to tell its format: enough for a PE header at its usual offsets. +const programHeadSize = 0x100 + +// programsIn reads the output of `git diff --raw -z --no-renames --diff-filter=AM` (": \0\0" per file) and names each added or modified regular file of the tree dir that is a compiled +// program, from its first bytes read locally. A symlink (mode 120000) and a submodule (160000) are not read. It never +// answers nil, so a caller can tell no programs from none reported. +func programsIn(dir, raw string) []Program { + out := []Program{} + f := strings.Split(raw, "\x00") + for i := 0; i+1 < len(f); i += 2 { + meta := strings.Fields(strings.TrimPrefix(f[i], ":")) + if len(meta) < 2 || (meta[1] != "100644" && meta[1] != "100755") { + continue + } + p := f[i+1] + full := filepath.Join(dir, filepath.FromSlash(p)) + if st, err := os.Lstat(full); err != nil || !st.Mode().IsRegular() { + continue + } + fh, err := os.Open(full) + if err != nil { + continue + } + head := make([]byte, programHeadSize) + n, _ := io.ReadFull(fh, head) + fh.Close() + if k := ExecutableKind(head[:n]); k != "" { + out = append(out, Program{Path: p, Kind: k}) + } + } + return out +} diff --git a/go/buildstep/programs_test.go b/go/buildstep/programs_test.go new file mode 100644 index 0000000..7cc26bc --- /dev/null +++ b/go/buildstep/programs_test.go @@ -0,0 +1,72 @@ +package buildstep + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func pe(offset int, magic string) []byte { + b := make([]byte, 0x100) + copy(b, "MZ") + b[0x3c] = byte(offset) + copy(b[offset:], magic) + return b +} + +func TestExecutableKindTellsProgramsFromText(t *testing.T) { // issue 1577 + for _, c := range []struct { + name string + head []byte + want string + }{ + {"elf", []byte("\x7fELF\x02\x01\x01"), "ELF"}, + {"mach-o 64", []byte{0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1}, "Mach-O"}, + {"mach-o 32 big", []byte{0xfe, 0xed, 0xfa, 0xce, 0, 0, 0, 7}, "Mach-O"}, + {"fat mach-o", []byte{0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 2}, "Mach-O"}, + {"java class", []byte{0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0x34}, ""}, + {"pe", pe(0x80, "PE\x00\x00"), "PE"}, + {"mz text", []byte("MZ is a text file starting with two letters"), ""}, + {"mz without a pe header", pe(0x80, "XX\x00\x00"), ""}, + {"text", []byte("package main"), ""}, + {"empty", nil, ""}, + } { + if got := ExecutableKind(c.head); got != c.want { + t.Errorf("%s: kind = %q, want %q", c.name, got, c.want) + } + } +} + +func TestProgramsInNamesTheAddedAndModifiedRegularFilesThatArePrograms(t *testing.T) { // issue 1577 + dir := t.TempDir() + write := func(name string, b []byte) { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), b, 0o755); err != nil { + t.Fatal(err) + } + } + write("m/prog", []byte("\x7fELF\x02\x01")) + write("m/notes", []byte("plain text")) + write("m/mac", []byte{0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1}) + if err := os.Symlink("prog", filepath.Join(dir, "m/link")); err != nil { + t.Fatal(err) + } + line := func(mode, path string) string { return ":000000 " + mode + " 0000 1111 A\x00" + path + "\x00" } + for _, c := range []struct { + name, raw string + want []Program + }{ + {"none", "", []Program{}}, + {"a program and a text", line("100755", "m/prog") + line("100644", "m/notes"), []Program{{"m/prog", "ELF"}}}, + {"two programs", line("100644", "m/prog") + line("100755", "m/mac"), []Program{{"m/prog", "ELF"}, {"m/mac", "Mach-O"}}}, + {"a symlink and a submodule are not read", line("120000", "m/link") + line("160000", "m/sub"), []Program{}}, + {"a path the tree lacks", line("100644", "m/gone"), []Program{}}, + } { + if got := programsIn(dir, c.raw); !reflect.DeepEqual(got, c.want) { + t.Errorf("%s: programs = %v, want %v", c.name, got, c.want) + } + } +} From f1d7c7377dc45606b9cba32456132b42de3b1d29 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:03:00 +0100 Subject: [PATCH 15/42] =?UTF-8?q?frombus=20reads=20the=20composing=20node'?= =?UTF-8?q?s=20own=20secrets=20sealed=20to=20it,=20and=20node=20principals?= =?UTF-8?q?=20may=20ask=20the=20vault's=20own=20in=20their=20own=20name=20?= =?UTF-8?q?(issue=201126,=20ADR=200550=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buscompose/buscompose_test.go | 27 ++++++++++++++++++++++++++- go/buscompose/nats.go | 18 ++++++++++++++++++ go/buscompose/users.go | 1 + go/compose/frombus/frombus.go | 16 ++++++++++++---- go/compose/frombus/frombus_test.go | 15 ++++++++++----- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 554c6a1..d1a3d8c 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -38,6 +38,15 @@ func beyondTheControllers(user string) map[string]bool { return out } +// mayBeyondTheControllers is what the copy may grant beyond the controller's where the records allow it, and need not +// where they do not: a node's ask of the vault's `own` while the vault serves it (issue 1126, ADR 0550 §5). +func mayBeyondTheControllers(user string) map[string]bool { + if _, ok := strings.CutPrefix(user, "node."); !ok { + return nil + } + return map[string]bool{VaultOwnCall(Principal{Kind: KindNode, Node: strings.TrimPrefix(user, "node.")}): true} +} + // quotedToken is a quoted token of a composed list. var quotedToken = regexp.MustCompile(`"[^"]*"`) @@ -89,7 +98,7 @@ func grantsAsTheControllers(t *testing.T, got, want string) { } } for g := range now { - if !was[g] && !intended[g] && !createdAdvisory.MatchString(`"`+g+`", `) { + if !was[g] && !intended[g] && !mayBeyondTheControllers(user)[g] && !createdAdvisory.MatchString(`"`+g+`", `) { t.Errorf("%s: the copy grants %s beyond the controller's, and no decision names it", user, g) } } @@ -194,6 +203,22 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { t.Error("a user of a seat not pulled by kind reads DEAD_LETTERS") } }) + // Issue 1126 (ADR 0550 §5): a node's engine asks the vault's `own` in its own name while the vault serves it, and no + // other node's. + t.Run("a node asks the vault its own secrets", func(t *testing.T) { + one := Principal{Kind: KindNode, Node: "one", AsksItsOwnSecrets: true} + perms, err := PermissionsFor(one) + if err != nil { + t.Fatal(err) + } + if !MayPublish(perms, VaultOwnCall(one)) || MayPublish(perms, VaultOwnCall(Principal{Kind: KindNode, Node: "two"})) { + t.Errorf("node one asks own as %v, and as two %v", MayPublish(perms, VaultOwnCall(one)), + MayPublish(perms, VaultOwnCall(Principal{Kind: KindNode, Node: "two"}))) + } + if plain, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one"}); MayPublish(plain, VaultOwnCall(one)) { + t.Error("a node asks own while the vault does not serve it") + } + }) t.Run("the controller's golden server configuration", func(t *testing.T) { seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} got, err := Compose( diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 8f5379b..067fbd4 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -213,6 +213,10 @@ type Principal struct { // StatesItsKeyToTheVault is a machine principal whose node-engine may ask the vault's `node-key` in its own name // (issue 786, novox/hq ADR 0372): set while the vault's holder serves it. That one verb, and nothing else new. StatesItsKeyToTheVault bool + // AsksItsOwnSecrets is a machine principal whose node-engine may ask the vault's `own` in its own name (novox/hq + // ADR 0547 §2, ADR 0550 §5, issue 1126): its modules' own secrets sealed to its key, for the compositions it makes + // itself. Set while the vault's holder serves it. + AsksItsOwnSecrets bool // FollowsGrantsUserList is the machine principal of the node holding mesh-broker: its node-engine reads the grants // part's sealed user list, one key and an object by name (novox/hq ADR 0547 clause 7, GrantsUserListFollows). FollowsGrantsUserList bool @@ -344,6 +348,10 @@ var VaultMoveIn = SeatVerb{Seat: "mesh-vault", Verb: "move-in"} // novox/hq ADR 0372); granted as Principal.StatesItsKeyToTheVault says. var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"} +// VaultOwn is the vault's verb a machine's node-engine asks in its own name for its modules' own secrets sealed to its +// key (issue 1126, ADR 0550 §5); granted as Principal.AsksItsOwnSecrets says. +var VaultOwn = SeatVerb{Seat: "mesh-vault", Verb: "own"} + // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the // controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release @@ -923,6 +931,11 @@ func PermissionsFor(p Principal) (Permissions, error) { if p.StatesItsKeyToTheVault { pub = append(pub, "mesh.seat."+VaultNodeKey.Seat+"."+CallKind+"."+VaultNodeKey.Verb+"."+CallerToken(p.Username())) } + // And, while the vault serves it, its modules' own secrets in its own name (issue 1126): sealed to its key, for the + // compositions it makes itself. + if p.AsksItsOwnSecrets { + pub = append(pub, VaultOwnCall(p)) + } // And, on the node holding mesh-broker, the grants part's sealed user list it follows from take-over (novox/hq ADR // 0547 clause 7): one key, and an object by its name. if p.FollowsGrantsUserList { @@ -1792,3 +1805,8 @@ const ( nodeToolsService = "node-tools" leaseKey = "holder" ) + +// VaultOwnCall is the subject a machine principal's node-engine asks the vault's `own` on, in its own name. +func VaultOwnCall(p Principal) string { + return "mesh.seat." + VaultOwn.Seat + "." + CallKind + "." + VaultOwn.Verb + "." + CallerToken(p.Username()) +} diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 472a0c4..56dbcc9 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -159,6 +159,7 @@ func Users(r Records) ([]Principal, error) { } out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks, MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey), + AsksItsOwnSecrets: servesVerb(r, VaultOwn), FollowsGrantsUserList: bus}) // **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one // per machine that runs it, delivered to the module running it. diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 0c7386a..ae1dba4 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -19,10 +19,11 @@ // - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the // caller composes every node (ADR 0547 §2, the prepare step's L4). // -// **Not yet read** (each a later subtask of issue 1126): licences, own secrets, issued credentials and core boxes, which -// the vault answers sealed to the node (ADR 0550 §3, §5). Without them the composition stands in for each and names it -// foreseen, as compose.Reading does for a value the Input does not hold. No node is joining before ADR 0547's third -// milestone (ADR 0550 §6), so Joining is always empty. +// **Own secrets** come from the vault sealed to the composing node's key (OwnSecrets), for that node alone. **Not yet +// read** (each a later subtask of issue 1126): licences, pair credentials, issued credentials and core boxes. Without one +// the composition stands in for it, and Compose refuses a composition that stands in for anything (compose.ErrStoodIn): +// a node never applies a stand-in. No node is joining before ADR 0547's third milestone (ADR 0550 §6), so Joining is +// always empty. package frombus import ( @@ -54,6 +55,9 @@ type Reader interface { // Values is one module's config values on one node as mesh-config resolved them (mesh-config's state `resolved`, // keys `..`), decoded; nil when it has none. Values(ctx context.Context, node, module string) (map[string]any, error) + // OwnSecrets is the own secrets of the composing node's modules, each sealed to that node's key, as the vault answers + // the node's engine (the vault's verb `own`, ADR 0550 §5); asked only for the node composing itself. + OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecretRow, error) } // Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the @@ -197,6 +201,10 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } + // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). + if in.OwnSecrets, err = r.OwnSecrets(ctx, s.Node); err != nil { + return compose.Input{}, fmt.Errorf("frombus: the own secrets of %s's modules: %w", s.Node, err) + } } return in, nil } diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 8e93364..8fdc0d5 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -26,6 +26,7 @@ type bus struct { manifests map[string][]byte values map[string]map[string]any // by node.module valuesRead []string + own []compose.OwnSecretRow } func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) { @@ -40,6 +41,8 @@ func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { } return raw, nil } +func (b *bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return b.own, nil } + func (b *bus) Values(_ context.Context, node, module string) (map[string]any, error) { b.valuesRead = append(b.valuesRead, node+"."+module) return b.values[node+"."+module], nil @@ -86,6 +89,7 @@ func TestInput(t *testing.T) { modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "unread": {}}, manifests: map[string][]byte{digest: xterm}, values: map[string]map[string]any{"laptop.xterm": {"font": "mono"}, "anchor.xterm": {"font": "serif"}}, + own: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, } } @@ -112,9 +116,10 @@ func TestInput(t *testing.T) { Ports: map[string]map[int]int{"xterm": {80: 8080}}, Settings: map[string][]catalogue.Layer{"xterm": {{From: "laptop", Values: map[string]any{"font": "mono"}}}}}, }, - Holdings: []catalogue.Held{{Claim: "mesh-terminal", Scope: "mesh", Node: "anchor", Module: "xterm"}}, - Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, - Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, + Holdings: []catalogue.Held{{Claim: "mesh-terminal", Scope: "mesh", Node: "anchor", Module: "xterm"}}, + OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, + Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, + Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, // The controller's order is by address; laptop enrolled under its peer's key, and a person named the other. CarriedPeers: []compose.CarriedPeer{{PublicKey: keys["laptop"], Address: "10.77.0.2", EnrolledAs: "laptop"}, {PublicKey: "peer", Address: "10.77.0.9", Named: "phone"}}, @@ -149,8 +154,8 @@ func TestInput(t *testing.T) { } b = mk() - if _, err := Input(ctx, b, Mesh{}, Scope{Every: true}); err != nil { - t.Fatal(err) + if every, err := Input(ctx, b, Mesh{}, Scope{Every: true}); err != nil || every.OwnSecrets != nil { + t.Fatalf("composing every node: %v, own secrets %v (the vault answers a node about itself alone)", err, every.OwnSecrets) } if !reflect.DeepEqual(b.valuesRead, []string{"anchor." + overlay.Name, "laptop." + overlay.Name, "laptop.xterm"}) { // Every node's placed modules, and nothing placed nowhere: anchor places no xterm. From dfe3ba94750c0813a8f3c38d98d20e22280f70f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:11:34 +0100 Subject: [PATCH 16/42] =?UTF-8?q?frombus=20exports=20a=20node's=20placemen?= =?UTF-8?q?t=20from=20a=20composition=20input,=20Input's=20inverse=20for?= =?UTF-8?q?=20what=20a=20person=20decides,=20and=20the=20controller's=20se?= =?UTF-8?q?at=20declares=20placement-export=20(issue=201126,=20ADR=200555?= =?UTF-8?q?=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/catalogue/verbs.go | 7 ++ go/compose/frombus/export.go | 100 +++++++++++++++++++++++++++++ go/compose/frombus/frombus_test.go | 20 +++++- 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 go/compose/frombus/export.go diff --git a/go/catalogue/verbs.go b/go/catalogue/verbs.go index 11f0b3e..9f980fb 100644 --- a/go/catalogue/verbs.go +++ b/go/catalogue/verbs.go @@ -126,6 +126,13 @@ var ControllerVerbs = []Verb{ {Name: "node", Description: "What one machine reported it can do, what it is assigned, and why.", Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"}), Replaces: []string{"hostnamectl", "uptime"}}, + {Name: "placement-export", Description: "Each machine's placement as the catalogue's placement file says it (novox/hq ADR 0555 " + + "§6, issue 1126): what a person decided of the machine (endpoint, site, hub, address on the private network, public " + + "domain, accounts, adoption, the tunnel found at adoption) and where its modules run (in order, with host ports, " + + "bindings, config placement layers, pins and holdings), read by the controller's own code from its own store, as JSON. " + + "Read-only; the one verb the controller gains for its exit (the operator, 2026-10-26 19:06:40), read once by the seed " + + "of the placement files.", + Input: schema(map[string]string{"node": "one machine's name; every machine when absent"}, nil)}, {Name: "modules", Description: "Every module the mesh holds: version, the commit it was built from, " + "and which machines run it.", Input: schema(nil, nil)}, diff --git a/go/compose/frombus/export.go b/go/compose/frombus/export.go new file mode 100644 index 0000000..996bb1e --- /dev/null +++ b/go/compose/frombus/export.go @@ -0,0 +1,100 @@ +package frombus + +import ( + "fmt" + "sort" + "strconv" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Export is one node's placement file as the composition input the controller fills says it (novox/hq ADR 0555 §6, +// issue 1126): the inverse of Input for what a person decides, so the controller's one read-only verb +// `placement-export` answers, from its own store by its own code, exactly what the seed writes into the catalogue. +// What the machine observes (its record) is not in it. +func Export(in compose.Input, node string) (delivery.Placement, error) { + var n *compose.Node + for i := range in.Nodes { + if in.Nodes[i].Name == node { + n = &in.Nodes[i] + } + } + if n == nil { + return delivery.Placement{}, fmt.Errorf("frombus: the input has no node %q", node) + } + p := delivery.Placement{Node: node, Decided: delivery.NodeDecided{ + Endpoint: n.Place.Endpoint, Site: n.Place.Site, Hub: n.Place.Hub, Address: n.Place.Address, + PublicDomain: n.PublicDomain, Account: n.Account, AccountHome: n.AccountHome, AgentAccount: n.AgentAccount, + AgentHome: n.AgentAccountHome, Adopted: n.Adopted, Taken: n.Taken}} + // The tunnel found at the node's adoption: the one its private network took over, or, on the hub, the mesh's adopted + // one whether or not the hub took it; and the names a person gave the hub's carried peers. + if t, ok := in.Tunnels[node]; ok { + rt := recordTunnelOf(t.Tunnel) + p.Decided.Tunnel = &rt + } else if in.Adopted != nil && in.Adopted.Hub == node { + rt := recordTunnelOf(in.Adopted.Tunnel) + p.Decided.Tunnel = &rt + } + if n.Place.Hub { + for _, c := range in.CarriedPeers { + if c.Named != "" { + if p.Decided.PeerNames == nil { + p.Decided.PeerNames = map[string]string{} + } + p.Decided.PeerNames[c.PublicKey] = c.Named + } + } + } + for _, module := range n.Assigned { + m := delivery.PlacedModule{Module: module} + for port, at := range n.Ports[module] { + if m.Ports == nil { + m.Ports = map[string]int{} + } + m.Ports[strconv.Itoa(port)] = at + } + for provision, to := range n.Bound[module] { + if m.Bindings == nil { + m.Bindings = map[string]delivery.Provider{} + } + m.Bindings[provision] = delivery.Provider{Node: to.Node, Module: to.Module} + } + for _, layer := range n.Settings[module] { + if layer.From == catalogue.MeshWideLayer { + m.HasMesh, m.MeshWide = true, layer.Placement + } else { + m.HasLayer, m.Layer = true, layer.Placement + } + } + p.Modules = append(p.Modules, m) + } + for provision, to := range n.Pins { + if p.Pins == nil { + p.Pins = map[string]delivery.Provider{} + } + p.Pins[provision] = delivery.Provider{Node: to.Node, Module: to.Module} + } + for _, h := range in.Holdings { + if h.Node == node { + p.Holds = append(p.Holds, delivery.Holding{Claim: h.Claim, Scope: h.Scope, Module: h.Module, Site: h.Site, Kind: h.Kind}) + } + } + sort.SliceStable(p.Holds, func(i, j int) bool { + if p.Holds[i].Claim != p.Holds[j].Claim { + return p.Holds[i].Claim < p.Holds[j].Claim + } + return p.Holds[i].Module < p.Holds[j].Module + }) + return p, nil +} + +func recordTunnelOf(t compose.Tunnel) delivery.RecordTunnel { + out := delivery.RecordTunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU, + Address: t.Address, Range: t.Range, PublicKey: t.PublicKey} + for _, peer := range t.Peers { + out.Peers = append(out.Peers, delivery.RecordTunnelPeer{PublicKey: peer.PublicKey, Address: peer.Address}) + } + return out +} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 8fdc0d5..5eeaafa 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -77,7 +77,7 @@ func TestInput(t *testing.T) { "anchor": {Node: "anchor", Decided: delivery.NodeDecided{Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1", Tunnel: tunnel, PeerNames: map[string]string{"peer": "phone"}}, Modules: []delivery.PlacedModule{{Module: overlay.Name}}, - Holds: []delivery.Holding{{Claim: "mesh-terminal", Scope: "mesh", Module: "xterm"}}}, + Holds: []delivery.Holding{{Claim: "mesh-private-network", Scope: "mesh", Module: overlay.Name}}}, "laptop": {Node: "laptop", Decided: delivery.NodeDecided{Endpoint: "laptop.example:51820", Site: "here", Address: "10.77.0.2"}, Modules: []delivery.PlacedModule{{Module: overlay.Name}, {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}}}, @@ -116,7 +116,7 @@ func TestInput(t *testing.T) { Ports: map[string]map[int]int{"xterm": {80: 8080}}, Settings: map[string][]catalogue.Layer{"xterm": {{From: "laptop", Values: map[string]any{"font": "mono"}}}}}, }, - Holdings: []catalogue.Held{{Claim: "mesh-terminal", Scope: "mesh", Node: "anchor", Module: "xterm"}}, + Holdings: []catalogue.Held{{Claim: "mesh-private-network", Scope: "mesh", Node: "anchor", Module: overlay.Name}}, OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, @@ -153,6 +153,22 @@ func TestInput(t *testing.T) { t.Fatalf("compositions differ or are empty: %d from the bus, %d from the store", len(fromBus), len(fromStore)) } + // ADR 0555 §6: Export is Input's inverse for what a person decides: the store's input exported, node by node, is the + // placement the bus holds, and read back it is the store's input again. + for node, placed := range mk().placements { + got, err := Export(want, node) + if err != nil { + t.Fatal(err) + } + if gj, _ := delivery.EncodePlacement(got); !delivery.SamePlacement(got, placed) { + pj, _ := delivery.EncodePlacement(placed) + t.Errorf("%s exported as %s, want %s", node, gj, pj) + } + } + if _, err := Export(want, "stranger"); err == nil { + t.Error("a node the input lacks was exported") + } + b = mk() if every, err := Input(ctx, b, Mesh{}, Scope{Every: true}); err != nil || every.OwnSecrets != nil { t.Fatalf("composing every node: %v, own secrets %v (the vault answers a node about itself alone)", err, every.OwnSecrets) From 4d5f43022453e68940d505c7ed3c82e7c7c6a955 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:13:56 +0100 Subject: [PATCH 17/42] =?UTF-8?q?A=20want=20the=20node=20composes=20itself?= =?UTF-8?q?=20says=20so=20(Self)=20and=20names=20the=20build=20alone;=20ev?= =?UTF-8?q?ery=20other=20want=20names=20its=20composition=20as=20before,?= =?UTF-8?q?=20so=20no=20caller=20of=20the=20old=20form=20changes=20(issue?= =?UTF-8?q?=201126,=20ADR=200547=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/assigner.go | 45 +++++++++++++++++++++---------- go/delivery/want_composed_test.go | 17 +++++++----- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/go/delivery/assigner.go b/go/delivery/assigner.go index 6c5d8e2..7987797 100644 --- a/go/delivery/assigner.go +++ b/go/delivery/assigner.go @@ -190,11 +190,14 @@ type Want struct { // of policy record that the node agent judged to recreate a container or move its data is applied only when its // want names the composition so allowed. Empty when none was asked. Yes string `json:"yes,omitempty"` + // Self says the node composes this module itself (ADR 0547 §2): the want names the build alone, and no composition. + // A want that is not Self names the one composition it allows (ADR 0539 clause 1), as before. + Self bool `json:"self,omitempty"` } func (w Want) check() error { if w.Removed { - if w.FP != "" || w.Composed != "" || w.Newest != "" || w.Rule != 0 || w.Backup != "" || w.Put || w.Hold { + if w.FP != "" || w.Composed != "" || w.Newest != "" || w.Rule != 0 || w.Backup != "" || w.Put || w.Hold || w.Self || w.Yes != "" { return fmt.Errorf("delivery: a removed want names no fingerprint, composition, rule, backup answer or put") } return nil @@ -208,9 +211,14 @@ func (w Want) check() error { if err := checkFingerprint(w.FP, "a want's fingerprint"); err != nil { return err } - // From ADR 0547's second milestone the node composes its own modules (§2), and a want names the build alone: no - // composition digest. A want written before names one, which is read as it was written. - if w.Composed != "" { + // From ADR 0547's second milestone a node the catalogue places composes its own modules (§2): its want is Self and + // names the build alone. Any other want names the one composition it allows (ADR 0539 clause 1). + switch { + case w.Self && w.Composed != "": + return fmt.Errorf("delivery: the want of %s says the node composes it, and names a composition", w.FP) + case !w.Self && w.Composed == "": + return fmt.Errorf("delivery: the want of %s names no composition: a want names the digest of the one composition it allows (ADR 0539 clause 1), unless the node composes it itself", w.FP) + case w.Composed != "": if err := checkDigest(w.Composed, "a want's composition digest"); err != nil { return err } @@ -320,15 +328,15 @@ type NodeWant struct { V int `json:"v"` Composed string `json:"composed"` Words string `json:"words,omitempty"` + // Self says the node composes its own composition (ADR 0547 §2) and the want names none. + Self bool `json:"self,omitempty"` } -// EncodeNodeWant is w as JSON at Version; it refuses a digest not of the sha256 form. From ADR 0547's second milestone -// the node composes its own composition (§2) and its own want names none: an empty Composed is that want. +// EncodeNodeWant is w as JSON at Version; it refuses a digest not of the sha256 form, and a want that names no +// composition unless it says the node composes its own (Self, ADR 0547 §2). func EncodeNodeWant(w NodeWant) ([]byte, error) { - if w.Composed != "" { - if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil { - return nil, err - } + if err := w.check(); err != nil { + return nil, err } w.V = Version return json.Marshal(w) @@ -346,14 +354,23 @@ func DecodeNodeWant(data []byte) (NodeWant, error) { if err := d.Decode(&w); err != nil { return NodeWant{}, fmt.Errorf("delivery: a node's own want does not fit its shape: %w", err) } - if w.Composed != "" { - if err := checkDigest(w.Composed, "the node's own want's composition digest"); err != nil { - return NodeWant{}, err - } + if err := w.check(); err != nil { + return NodeWant{}, err } return w, nil } +// check refuses a node's own want that names no composition and does not say the node composes its own, or does both. +func (w NodeWant) check() error { + switch { + case w.Self && w.Composed != "": + return fmt.Errorf("delivery: the node's own want says the node composes it, and names a composition") + case w.Self: + return nil + } + return checkDigest(w.Composed, "the node's own want's composition digest") +} + // EncodeHeld is h as JSON at Version; it refuses a node name a key refuses. func EncodeHeld(h Held) ([]byte, error) { if err := h.check(); err != nil { diff --git a/go/delivery/want_composed_test.go b/go/delivery/want_composed_test.go index 98d3476..72858db 100644 --- a/go/delivery/want_composed_test.go +++ b/go/delivery/want_composed_test.go @@ -19,8 +19,8 @@ func TestAWantNamesItsComposition(t *testing.T) { {FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: RuleComposed, Words: "no composition of b for ace yet"}, {Removed: true}, // ADR 0547 §2, §3 (issue 1126): from the second milestone a want names the build alone, and the yes a person gave. - {FP: fp2of('a')}, - {FP: fp2of('a'), Yes: hashOf('3')}, + {FP: fp2of('a'), Self: true}, + {FP: fp2of('a'), Self: true, Yes: hashOf('3')}, } { raw, err := EncodeWant(w) if err != nil { @@ -36,7 +36,10 @@ func TestAWantNamesItsComposition(t *testing.T) { w Want why string }{ - {Want{FP: fp2of('a'), Yes: "sha256:12"}, "a want's yes allows"}, + {Want{FP: fp2of('a')}, "names no composition"}, + {Want{FP: fp2of('a'), Self: true, Composed: hashOf('1')}, "names a composition"}, + {Want{FP: fp2of('a'), Self: true, Yes: "sha256:12"}, "a want's yes allows"}, + {Want{Removed: true, Self: true}, "removed want names no"}, {Want{FP: fp2of('a'), Composed: "sha256:12"}, "not sha256"}, {Want{Removed: true, Composed: hashOf('1')}, "removed want names no"}, {Want{FP: fp2of('a'), Composed: hashOf('1'), Backup: "later"}, `backup answer "later"`}, @@ -48,8 +51,8 @@ func TestAWantNamesItsComposition(t *testing.T) { t.Errorf("EncodeWant(%+v) = %v, want a refusal saying %q", r.w, err, r.why) } } - if w, err := DecodeWant([]byte(`{"v":1,"fp":"` + fp2of('a') + `"}`)); err != nil || w.Composed != "" { - t.Errorf("a want naming the build alone (ADR 0547 §2) is not read: %+v, %v", w, err) + if _, err := DecodeWant([]byte(`{"v":1,"fp":"` + fp2of('a') + `"}`)); err == nil || !strings.Contains(err.Error(), "names no composition") { + t.Errorf("a want of release 5, naming no composition and not the node's own, was read: %v", err) } if RuleComposed != 7 { t.Errorf("RuleComposed = %d, want the seventh rule", RuleComposed) @@ -90,13 +93,13 @@ func TestTheNodesOwnWant(t *testing.T) { if err != nil || back != w { t.Errorf("node want came back %+v, %v", back, err) } - for _, bad := range []NodeWant{{Composed: "sha256:1"}} { + for _, bad := range []NodeWant{{}, {Composed: "sha256:1"}, {Self: true, Composed: hashOf('2')}} { if _, err := EncodeNodeWant(bad); err == nil { t.Errorf("EncodeNodeWant(%+v) accepted", bad) } } // ADR 0547 §2 (issue 1126): from the second milestone the node composes its own composition, and its want names none. - if raw, err := EncodeNodeWant(NodeWant{Words: "the node composes its own"}); err != nil { + if raw, err := EncodeNodeWant(NodeWant{Self: true, Words: "the node composes its own"}); err != nil { t.Errorf("a node want naming no composition: %v", err) } else if back, err := DecodeNodeWant(raw); err != nil || back.Composed != "" { t.Errorf("a node want naming no composition came back %+v, %v", back, err) From 6447a1c6d4515193845b39a18c8643b006f8598a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:17:45 +0100 Subject: [PATCH 18/42] =?UTF-8?q?placement-seed=20writes=20each=20node's?= =?UTF-8?q?=20first=20placement=20file=20from=20the=20controller's=20expor?= =?UTF-8?q?t,=20checked=20before=20anything=20is=20written,=20once=20(issu?= =?UTF-8?q?e=201126,=20ADR=200549=20=C2=A74,=20ADR=200555=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/cmd/placement-seed/main.go | 139 +++++++++++++++++++++++++++++ go/cmd/placement-seed/main_test.go | 47 ++++++++++ go/delivery/delivery_test.go | 3 +- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 go/cmd/placement-seed/main.go create mode 100644 go/cmd/placement-seed/main_test.go diff --git a/go/cmd/placement-seed/main.go b/go/cmd/placement-seed/main.go new file mode 100644 index 0000000..93fae61 --- /dev/null +++ b/go/cmd/placement-seed/main.go @@ -0,0 +1,139 @@ +// placement-seed writes the catalogue's first placement files (novox/hq ADR 0549 §4, ADR 0555 §6, issue 1126) from the +// answer of the controller's one read-only verb `placement-export`: one `placement/.json` per node, each the +// node's placement as the controller's own code read it from its own store, checked as the catalogue's check checks it +// (delivery.CheckPlacements) before anything is written. The pull request it makes merges only when its composition check +// composes every node equal to what it runs. +// +// placement-seed -catalogue < export.json +// placement-seed -catalogue -export export.json +// +// export.json is the verb's answer as it prints it ({"": , ...}, or the verb's wrapping with it under +// "answer"). It opens no network connection, bus or store; it writes only /placement/.json, and refuses to +// write over a placement file already there. Exit codes: 0 written; 1 the export cannot be read, is refused by the +// check, or a file is there; 2 the flags are wrong. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func main() { + catalogue := flag.String("catalogue", "", "the checkout of the catalogue to write placement/.json into") + export := flag.String("export", "", "the verb's answer; standard input when absent") + flag.Parse() + if *catalogue == "" || flag.NArg() > 0 { + fmt.Fprintln(os.Stderr, "usage: placement-seed -catalogue [-export ]") + os.Exit(2) + } + in := io.Reader(os.Stdin) + if *export != "" { + f, err := os.Open(*export) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer f.Close() + in = f + } + written, err := Seed(in, *catalogue) + for _, path := range written { + fmt.Println(path) + } + if err != nil { + fmt.Fprintln(os.Stderr, "placement-seed:", err) + os.Exit(1) + } +} + +// Seed reads the export and writes each node's placement file under root, answering the paths written. Nothing is +// written when the export does not read, the check refuses it, or a placement file is already there. +func Seed(export io.Reader, root string) ([]string, error) { + raw, err := io.ReadAll(export) + if err != nil { + return nil, err + } + placements, err := read(raw) + if err != nil { + return nil, err + } + all := func(string) bool { return true } + if refused := delivery.CheckPlacements(placements, all, all); len(refused) > 0 { + return nil, fmt.Errorf("the export is refused by the catalogue's check: %v", refused) + } + nodes := make([]string, 0, len(placements)) + for node := range placements { + nodes = append(nodes, node) + } + sort.Strings(nodes) + files := map[string][]byte{} + for _, node := range nodes { + p := placements[node] + p.Revision = "" // only the indexer writes a revision, into the key + if _, err := delivery.EncodePlacement(p); err != nil { + return nil, fmt.Errorf("%s: %w", node, err) + } + p.V = delivery.Version + body, err := json.MarshalIndent(p, "", " ") + if err != nil { + return nil, err + } + path := filepath.Join(root, filepath.FromSlash(delivery.PlacementFile(node))) + if _, err := os.Stat(path); err == nil { + return nil, fmt.Errorf("%s is already there; the seed is written once (ADR 0549 §4)", path) + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + files[path] = append(body, '\n') + } + var written []string + for _, node := range nodes { + path := filepath.Join(root, filepath.FromSlash(delivery.PlacementFile(node))) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return written, err + } + if err := os.WriteFile(path, files[path], 0o644); err != nil { + return written, err + } + written = append(written, path) + } + return written, nil +} + +// read is the export as the verb prints it, or wrapped as a verb's answer (the controller's verbAnswer: its JSON under +// "answer"). +func read(raw []byte) (map[string]delivery.Placement, error) { + var wrapped struct { + Answer json.RawMessage `json:"answer"` + } + if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Answer) > 0 { + raw = wrapped.Answer + } + var byNode map[string]json.RawMessage + if err := json.Unmarshal(raw, &byNode); err != nil { + return nil, fmt.Errorf("the export is not {: }: %w", err) + } + if len(byNode) == 0 { + return nil, errors.New("the export names no node") + } + out := map[string]delivery.Placement{} + for node, body := range byNode { + p, err := delivery.DecodePlacement(body) + if err != nil { + return nil, fmt.Errorf("%s: %w", node, err) + } + if p.Node != node { + return nil, fmt.Errorf("the export's placement of %s names the node %q", node, p.Node) + } + out[node] = p + } + return out, nil +} diff --git a/go/cmd/placement-seed/main_test.go b/go/cmd/placement-seed/main_test.go new file mode 100644 index 0000000..6dd4399 --- /dev/null +++ b/go/cmd/placement-seed/main_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Issue 1126 (ADR 0549 §4, ADR 0555 §6): the seed writes one placement file per node of the export, as the catalogue's +// check and the indexer read it, from the verb's answer bare or wrapped; it writes nothing for an export the check +// refuses, and never over a file already there. +func TestTheSeedWritesEachNodesPlacementFileOnce(t *testing.T) { + export := `{"answer":{"ace":{"v":1,"node":"ace","revision":"x","decided":{"address":"10.77.0.2"},"modules":[{"module":"plex","ports":{"32400":32400}}]},` + + `"novox":{"v":1,"node":"novox","decided":{"hub":true,"address":"10.77.0.1"},"modules":[]}}}` + root := t.TempDir() + written, err := Seed(strings.NewReader(export), root) + if err != nil || len(written) != 2 { + t.Fatalf("written %v, %v", written, err) + } + raw, err := os.ReadFile(filepath.Join(root, "placement", "ace.json")) + if err != nil { + t.Fatal(err) + } + p, err := delivery.DecodePlacement(raw) + if err != nil || p.Node != "ace" || p.Revision != "" || p.HostPorts()["plex"][32400] != 32400 { + t.Errorf("ace's file reads %+v, %v", p, err) + } + if _, err := Seed(strings.NewReader(export), root); err == nil || !strings.Contains(err.Error(), "already there") { + t.Errorf("a second seed over the files: %v", err) + } + clash := `{"ace":{"v":1,"node":"ace","decided":{"address":"10.77.0.1"},"modules":[]},"novox":{"v":1,"node":"novox","decided":{"address":"10.77.0.1"},"modules":[]}}` + other := t.TempDir() + if _, err := Seed(strings.NewReader(clash), other); err == nil || !strings.Contains(err.Error(), "both have the address") { + t.Errorf("an export the check refuses: %v", err) + } + if entries, _ := os.ReadDir(filepath.Join(other, "placement")); len(entries) != 0 { + t.Errorf("a refused export wrote %v", entries) + } + for _, bad := range []string{`[]`, `{}`, `{"ace":{"v":1,"node":"novox","modules":[]}}`} { + if _, err := Seed(strings.NewReader(bad), t.TempDir()); err == nil { + t.Errorf("%s was seeded", bad) + } + } +} diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index 2f3c480..a831685 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -411,6 +411,7 @@ func testPulseTTL(t *testing.T) { // 1131/AC8: equal values say same, different ones do not. func TestSame(t *testing.T) { + t.Run("have removed", testHaveRemoved) a := Have{V: 1, Build: "b", Phase: PhaseRunning, Health: HealthOK} b := a if !SameHave(a, b) { @@ -464,7 +465,7 @@ func TestSame(t *testing.T) { } } -func TestHaveRemoved(t *testing.T) { // 1370/AC3 +func testHaveRemoved(t *testing.T) { // 1370/AC3 in := Have{Build: "b1", Phase: PhaseRemoved, Health: HealthUnknown, Detail: "the declaration no longer holds it"} b, err := EncodeHave(in) if err != nil { From 91413de335168205dbe5f958aad2e9516cb0a1be Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:18:46 +0100 Subject: [PATCH 19/42] =?UTF-8?q?compose=20names=20the=20pair=20credential?= =?UTF-8?q?s=20a=20node's=20composition=20reads=20as=20consumer=20and=20pr?= =?UTF-8?q?ovider,=20and=20frombus=20asks=20them=20sealed=20before=20it=20?= =?UTF-8?q?composes=20(issue=201126,=20ADR=200550=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/credentials_needed.go | 64 ++++++++++++++++++++++++++++++ go/compose/credentials_test.go | 9 +++++ go/compose/frombus/frombus.go | 19 ++++++++- go/compose/frombus/frombus_test.go | 4 ++ 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 go/compose/credentials_needed.go diff --git a/go/compose/credentials_needed.go b/go/compose/credentials_needed.go new file mode 100644 index 0000000..a3a9bc1 --- /dev/null +++ b/go/compose/credentials_needed.go @@ -0,0 +1,64 @@ +package compose + +import ( + "fmt" + "sort" +) + +// CredentialKey names one pair credential by the ends it joins: the provision, the consumer's node and module, the +// provider's node, and the local name where a module keeps several (Credential's and Issued's key). +type CredentialKey struct { + Provision string `json:"provision"` + Consumer string `json:"consumer"` + ConsumerModule string `json:"consumer_module"` + Provider string `json:"provider"` + Local string `json:"local,omitempty"` +} + +// CredentialsNeeded is every pair credential node's composition reads (novox/hq ADR 0547 §2, ADR 0550 §5, issue 1126): +// as a consumer, each need of its resolution a provider's credential answers (a need answered by a record is a licence's, +// not a pair's); as a provider, each credential it must carry a grant for (rowsFor). A node that composes its own asks the +// vault for exactly these, sealed to its key, and composes with them; one it does not get is stood in for, which the +// node's own apply refuses (CompositionsApplied). +func CredentialsNeeded(in Input, node string) (asConsumer, asProvider []CredentialKey, err error) { + c := newComposer(in) + resolved, err := c.resolution(node) + if err != nil { + return nil, nil, err + } + for _, n := range resolved.Needs { + if n.ByRecord || n.From == "" { + continue + } + asConsumer = append(asConsumer, CredentialKey{Provision: n.Name, Consumer: node, ConsumerModule: n.For, + Provider: n.From, Local: n.Local}) + } + rows, err := c.rowsFor(node) + if err != nil { + return nil, nil, fmt.Errorf("compose: the credentials %s provides: %w", node, err) + } + for _, r := range rows { + asProvider = append(asProvider, CredentialKey{Provision: r.Provision, Consumer: r.Consumer, + ConsumerModule: r.ConsumerModule, Provider: node, Local: r.Local}) + } + sortKeys(asConsumer) + sortKeys(asProvider) + return asConsumer, asProvider, nil +} + +func sortKeys(ks []CredentialKey) { + sort.Slice(ks, func(i, j int) bool { + a, b := ks[i], ks[j] + switch { + case a.Provision != b.Provision: + return a.Provision < b.Provision + case a.Provider != b.Provider: + return a.Provider < b.Provider + case a.Consumer != b.Consumer: + return a.Consumer < b.Consumer + case a.ConsumerModule != b.ConsumerModule: + return a.ConsumerModule < b.ConsumerModule + } + return a.Local < b.Local + }) +} diff --git a/go/compose/credentials_test.go b/go/compose/credentials_test.go index 924df06..0697358 100644 --- a/go/compose/credentials_test.go +++ b/go/compose/credentials_test.go @@ -172,6 +172,15 @@ func aGrantMesh(t *testing.T) Input { // the sealed copy a stand-in is composed and named (novox/hq plan.go grantsFor). func TestAGrantIsCreatedForEachCredentialMadeForAConsumer(t *testing.T) { in := aGrantMesh(t) + // 1126 (ADR 0550 §5): the credentials each node's composition reads, by their ends: laptop's files consumes anchor's + // s3-bucket, and anchor grants it; the node composing its own asks the vault for exactly these. + pair := CredentialKey{Provision: "s3-bucket", Consumer: "laptop", ConsumerModule: "files", Provider: "anchor"} + if consumer, provider, err := CredentialsNeeded(in, "laptop"); err != nil || len(provider) != 0 || len(consumer) != 1 || consumer[0] != pair { + t.Errorf("laptop needs %v as consumer and %v as provider (%v), want %v as consumer", consumer, provider, err, pair) + } + if consumer, provider, err := CredentialsNeeded(in, "anchor"); err != nil || len(consumer) != 0 || len(provider) != 1 || provider[0] != pair { + t.Errorf("anchor needs %v as consumer and %v as provider (%v), want %v as provider", consumer, provider, err, pair) + } got := rendered(t, in, "anchor", Foreseeing) if len(got.Rendering.Grants) != 1 { t.Fatalf("grants are %+v", got.Rendering.Grants) diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index ae1dba4..1bc86f5 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -19,8 +19,9 @@ // - each module's config values on a node: mesh-config's resolved state, the composing node's slice alone unless the // caller composes every node (ADR 0547 §2, the prepare step's L4). // -// **Own secrets** come from the vault sealed to the composing node's key (OwnSecrets), for that node alone. **Not yet -// read** (each a later subtask of issue 1126): licences, pair credentials, issued credentials and core boxes. Without one +// **Own secrets and pair credentials** come from the vault sealed to the composing node's key (OwnSecrets, Credentials), +// for that node alone: as a consumer and as a provider, exactly the keys compose.CredentialsNeeded names. **Not yet +// read** (each a later subtask of issue 1126): licences (none is placed today) and core boxes. Without one // the composition stands in for it, and Compose refuses a composition that stands in for anything (compose.ErrStoodIn): // a node never applies a stand-in. No node is joining before ADR 0547's third milestone (ADR 0550 §6), so Joining is // always empty. @@ -58,6 +59,10 @@ type Reader interface { // OwnSecrets is the own secrets of the composing node's modules, each sealed to that node's key, as the vault answers // the node's engine (the vault's verb `own`, ADR 0550 §5); asked only for the node composing itself. OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecretRow, error) + // Credentials is, for the composing node, each pair credential it reads as a consumer, held and sealed to its key, + // and each it grants as a provider, with the copy sealed to it (the vault's `own` with the keys, ADR 0550 §5). A key + // the vault holds no value for is left out; the composition then stands in for it, and Compose refuses it. + Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) } // Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the @@ -217,6 +222,16 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s if err != nil { return nil, nil, err } + // The pair credentials its composition reads, asked of the vault sealed to it (ADR 0550 §5). + asConsumer, asProvider, err := compose.CredentialsNeeded(in, s.Node) + if err != nil { + return nil, nil, err + } + if len(asConsumer)+len(asProvider) > 0 { + if in.Credentials, in.Issued, err = r.Credentials(ctx, s.Node, asConsumer, asProvider); err != nil { + return nil, nil, fmt.Errorf("frombus: the credentials of %s: %w", s.Node, err) + } + } return compose.CompositionsApplied(in, s.Node, fps) } diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 5eeaafa..c6c5642 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -43,6 +43,10 @@ func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { } func (b *bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return b.own, nil } +func (b *bus) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + return nil, nil, nil +} + func (b *bus) Values(_ context.Context, node, module string) (map[string]any, error) { b.valuesRead = append(b.valuesRead, node+"."+module) return b.values[node+"."+module], nil From 7d81e8326fc15088024c9f154f0e22ff84b549c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:22:58 +0100 Subject: [PATCH 20/42] =?UTF-8?q?frombus=20gives=20the=20composing=20node'?= =?UTF-8?q?s=20core=20credentials=20as=20the=20vault's=20boxes=20sealed=20?= =?UTF-8?q?to=20it,=20from=20the=20same=20own=20secrets=20(issue=201126,?= =?UTF-8?q?=20ADR=200349=20=C2=A716,=20ADR=200550=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/frombus/frombus.go | 8 ++++++++ go/compose/frombus/frombus_test.go | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 1bc86f5..44cde19 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -210,6 +210,14 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if in.OwnSecrets, err = r.OwnSecrets(ctx, s.Node); err != nil { return compose.Input{}, fmt.Errorf("frombus: the own secrets of %s's modules: %w", s.Node, err) } + // The core's credentials travel as the vault's box sealed to the machine (ADR 0349 §16): the same values, which + // the composition reads for a core module as its kept boxes. + for _, o := range in.OwnSecrets { + if json.Valid([]byte(o.Sealed)) { + in.KeptCoreBoxes = append(in.KeptCoreBoxes, compose.CoreBox{Node: o.Node, Module: o.Module, Name: o.Name, + SealedTo: o.SealedTo, Box: json.RawMessage(o.Sealed)}) + } + } } return in, nil } diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index c6c5642..fedc381 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -122,8 +122,10 @@ func TestInput(t *testing.T) { }, Holdings: []catalogue.Held{{Claim: "mesh-private-network", Scope: "mesh", Node: "anchor", Module: overlay.Name}}, OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, - Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, - Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, + // The same value as a core credential's box, for a core module (ADR 0349 §16). + KeptCoreBoxes: []compose.CoreBox{{Node: "laptop", Module: "xterm", Name: "token", SealedTo: "sk", Box: json.RawMessage("{}")}}, + Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, + Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, // The controller's order is by address; laptop enrolled under its peer's key, and a person named the other. CarriedPeers: []compose.CarriedPeer{{PublicKey: keys["laptop"], Address: "10.77.0.2", EnrolledAs: "laptop"}, {PublicKey: "peer", Address: "10.77.0.9", Named: "phone"}}, From 944a35d200c9b22ad6a81cadd06c45ee64bd9f8f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:23:06 +0100 Subject: [PATCH 21/42] =?UTF-8?q?A=20node's=20record=20carries=20the=20mes?= =?UTF-8?q?h's=20own=20certificate=20as=20the=20machine=20holds=20it,=20an?= =?UTF-8?q?d=20frombus=20composes=20with=20the=20composing=20node's=20(iss?= =?UTF-8?q?ue=201126,=20ADR=200555=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/frombus/frombus.go | 14 ++++++++++++++ go/compose/frombus/frombus_test.go | 3 ++- go/delivery/noderecord.go | 3 +++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 44cde19..67f2c1e 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -106,6 +106,20 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er in := compose.Input{OverlayCIDR: mesh.OverlayCIDR, InternalSuffix: mesh.InternalSuffix, Authority: mesh.Authority, Catalogue: map[string]catalogue.Manifest{}} + if in.Authority == "" { + // The mesh's own certificate, public, as the composing node holds it; for the check of every node, as the first + // node holding one does. + if rec, ok := records[s.Node]; ok && rec.Authority != "" { + in.Authority = rec.Authority + } else { + for _, node := range sortedKeys(records) { + if a := records[node].Authority; a != "" { + in.Authority = a + break + } + } + } + } read := func(module, digest string) (catalogue.Manifest, error) { raw, err := r.Manifest(ctx, digest) if err != nil { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index fedc381..b8ca0ef 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -88,7 +88,7 @@ func TestInput(t *testing.T) { }, records: map[string]delivery.NodeRecord{ "anchor": {Node: "anchor", TunnelKey: keys["anchor"], Capabilities: caps, Contract: 2}, - "laptop": {Node: "laptop", TunnelKey: keys["laptop"], Capabilities: caps}, + "laptop": {Node: "laptop", TunnelKey: keys["laptop"], Capabilities: caps, Authority: "the mesh's certificate"}, }, modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "unread": {}}, manifests: map[string][]byte{digest: xterm}, @@ -127,6 +127,7 @@ func TestInput(t *testing.T) { Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, // The controller's order is by address; laptop enrolled under its peer's key, and a person named the other. + Authority: "the mesh's certificate", CarriedPeers: []compose.CarriedPeer{{PublicKey: keys["laptop"], Address: "10.77.0.2", EnrolledAs: "laptop"}, {PublicKey: "peer", Address: "10.77.0.9", Named: "phone"}}, } diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go index d7fef79..c22f8b2 100644 --- a/go/delivery/noderecord.go +++ b/go/delivery/noderecord.go @@ -52,6 +52,9 @@ type NodeRecord struct { Networks []catalogue.AttachedNetwork `json:"networks,omitempty"` NetworksSaid bool `json:"networks-said,omitempty"` Contract int `json:"contract,omitempty"` + // Authority is the mesh's own certificate as this machine holds it (public): the one its declaration last wrote where + // a module asked for it. A node composing its own composes a certificate with it (compose.Input.Authority). + Authority string `json:"authority,omitempty"` } func (r NodeRecord) check() error { From f55b34febc2d78632b73ecc547d9173d6cf24752 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:25:09 +0100 Subject: [PATCH 22/42] =?UTF-8?q?A=20node=20composing=20its=20own=20hears?= =?UTF-8?q?=20the=20vault's=20secret-changed,=20names=20and=20generations?= =?UTF-8?q?=20only=20(issue=201126,=20ADR=200547=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buscompose/buscompose_test.go | 7 ++++++- go/buscompose/nats.go | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index d1a3d8c..f3705d2 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -1,6 +1,7 @@ package buscompose import ( + "slices" "encoding/json" "os" "path/filepath" @@ -44,7 +45,8 @@ func mayBeyondTheControllers(user string) map[string]bool { if _, ok := strings.CutPrefix(user, "node."); !ok { return nil } - return map[string]bool{VaultOwnCall(Principal{Kind: KindNode, Node: strings.TrimPrefix(user, "node.")}): true} + return map[string]bool{VaultOwnCall(Principal{Kind: KindNode, Node: strings.TrimPrefix(user, "node.")}): true, + VaultSecretChanged: true} } // quotedToken is a quoted token of a composed list. @@ -218,6 +220,9 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { if plain, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one"}); MayPublish(plain, VaultOwnCall(one)) { t.Error("a node asks own while the vault does not serve it") } + if !slices.Contains(perms.Subscribe, VaultSecretChanged) { + t.Errorf("node one does not hear the vault's secret-changed: %v", perms.Subscribe) + } }) t.Run("the controller's golden server configuration", func(t *testing.T) { seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 067fbd4..54a04da 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -935,6 +935,9 @@ func PermissionsFor(p Principal) (Permissions, error) { // compositions it makes itself. if p.AsksItsOwnSecrets { pub = append(pub, VaultOwnCall(p)) + // And the vault's word that a secret changed (names and generations, never a value), so the node composes again + // on it rather than at the next unrelated change (ADR 0547 §2). + sub = append(sub, VaultSecretChanged) } // And, on the node holding mesh-broker, the grants part's sealed user list it follows from take-over (novox/hq ADR // 0547 clause 7): one key, and an object by its name. @@ -1810,3 +1813,7 @@ const ( func VaultOwnCall(p Principal) string { return "mesh.seat." + VaultOwn.Seat + "." + CallKind + "." + VaultOwn.Verb + "." + CallerToken(p.Username()) } + +// VaultSecretChanged is the vault's event that a secret changed (a name and its generation, never a value), which a +// node composing its own hears to compose again (issue 1126, ADR 0547 §2). +const VaultSecretChanged = "mesh.mod." + VaultOwn.Seat + ".event.secret-changed" From 53c2def1a5200792ccbc9b660172cc198fb0764c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:25:19 +0100 Subject: [PATCH 23/42] VaultSecretChanged is a variable (its seat is), and gofmt (issue 1126) --- go/buscompose/buscompose_test.go | 2 +- go/buscompose/nats.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index f3705d2..6bb593d 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -1,12 +1,12 @@ package buscompose import ( - "slices" "encoding/json" "os" "path/filepath" "reflect" "regexp" + "slices" "strings" "testing" ) diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 54a04da..b548eea 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -1816,4 +1816,4 @@ func VaultOwnCall(p Principal) string { // VaultSecretChanged is the vault's event that a secret changed (a name and its generation, never a value), which a // node composing its own hears to compose again (issue 1126, ADR 0547 §2). -const VaultSecretChanged = "mesh.mod." + VaultOwn.Seat + ".event.secret-changed" +var VaultSecretChanged = "mesh.mod." + VaultOwn.Seat + ".event.secret-changed" From 9713c3023a18b211499d90a077fe8e677f968d86 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:38:37 +0100 Subject: [PATCH 24/42] =?UTF-8?q?Placement=20carries=20a=20settings-form?= =?UTF-8?q?=20module's=20config=20values,=20which=20mesh-config=20does=20n?= =?UTF-8?q?ot=20hold,=20and=20the=20export=20writes=20them=20(issue=201126?= =?UTF-8?q?,=20ADR=200555=20=C2=A71):=20the=20media=20modules=20on=20one?= =?UTF-8?q?=20node=20would=20compose=20without=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/compose/frombus/export.go | 9 +++++++ go/compose/frombus/frombus_test.go | 40 +++++++++++++++++++++--------- go/delivery/placement.go | 38 ++++++++++++++++++++-------- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/go/compose/frombus/export.go b/go/compose/frombus/export.go index 996bb1e..ad81a77 100644 --- a/go/compose/frombus/export.go +++ b/go/compose/frombus/export.go @@ -61,11 +61,20 @@ func Export(in compose.Input, node string) (delivery.Placement, error) { } m.Bindings[provision] = delivery.Provider{Node: to.Node, Module: to.Module} } + // A module in the settings form keeps its config values in its layers, not in mesh-config: they are exported with + // the placement. A module in the config form has mesh-config's values laid into its layer, which are not. + settingsForm := !in.Catalogue[module].DeclaresConfig() for _, layer := range n.Settings[module] { if layer.From == catalogue.MeshWideLayer { m.HasMesh, m.MeshWide = true, layer.Placement + if settingsForm && len(layer.Values) > 0 { + m.MeshValues = layer.Values + } } else { m.HasLayer, m.Layer = true, layer.Placement + if settingsForm && len(layer.Values) > 0 { + m.Values = layer.Values + } } } p.Modules = append(p.Modules, m) diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index b8ca0ef..d5ec534 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -71,6 +71,12 @@ func TestInput(t *testing.T) { t.Fatal(err) } digest := delivery.ManifestDigest(xterm) + // A module in the settings form (plex and the media modules on one node, 2026-10-26): its config values are in its + // layers, not mesh-config, so the placement carries them (MeshValues, Values) and mesh-config's are not read for it. + media := []byte(`{"module":"media","version":"1","settings":{"lines":{"kind":"preference","default":0,"why":"kept"},` + + `"name":{"kind":"preference","default":"x","why":"shown"}},"resources":[{"id":"conf","type":"file",` + + `"path":"/srv/media/conf","mode":"0644","content":"uid=${setting:lines} name=${setting:name}\n"}]}`) + mediaDigest := delivery.ManifestDigest(media) keys := map[string]string{"anchor": publicKey(t), "laptop": publicKey(t)} caps := []string{"container-runtime", "systemd", "wireguard"} tunnel := &delivery.RecordTunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", @@ -84,16 +90,19 @@ func TestInput(t *testing.T) { Holds: []delivery.Holding{{Claim: "mesh-private-network", Scope: "mesh", Module: overlay.Name}}}, "laptop": {Node: "laptop", Decided: delivery.NodeDecided{Endpoint: "laptop.example:51820", Site: "here", Address: "10.77.0.2"}, Modules: []delivery.PlacedModule{{Module: overlay.Name}, - {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}}}, + {Module: "xterm", HasLayer: true, Ports: map[string]int{"80": 8080}}, + {Module: "media", HasMesh: true, MeshValues: map[string]any{"name": "mesh"}, HasLayer: true, + Layer: map[string]any{"places": map[string]any{"data": "/srv/data"}}, Values: map[string]any{"lines": float64(1000)}}}}, }, records: map[string]delivery.NodeRecord{ "anchor": {Node: "anchor", TunnelKey: keys["anchor"], Capabilities: caps, Contract: 2}, "laptop": {Node: "laptop", TunnelKey: keys["laptop"], Capabilities: caps, Authority: "the mesh's certificate"}, }, - modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "unread": {}}, - manifests: map[string][]byte{digest: xterm}, - values: map[string]map[string]any{"laptop.xterm": {"font": "mono"}, "anchor.xterm": {"font": "serif"}}, - own: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, + modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "media": {Manifest: mediaDigest}, "unread": {}}, + manifests: map[string][]byte{digest: xterm, mediaDigest: media}, + values: map[string]map[string]any{"laptop.xterm": {"font": "mono"}, "anchor.xterm": {"font": "serif"}, + "laptop.media": {"lines": "not mesh-config's"}}, + own: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, } } @@ -107,18 +116,24 @@ func TestInput(t *testing.T) { if err != nil { t.Fatal(err) } + parsedMedia, err := catalogue.ParseManifestForInputs(media) + if err != nil { + t.Fatal(err) + } capsSet := map[string]bool{"container-runtime": true, "systemd": true, "wireguard": true} tun := compose.Tunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", Peers: []compose.TunnelPeer{{PublicKey: "peer", Address: "10.77.0.9/32"}, {PublicKey: keys["laptop"], Address: "10.77.0.2/32"}}} want := compose.Input{ - Catalogue: map[string]catalogue.Manifest{overlay.Name: private, "xterm": parsed}, + Catalogue: map[string]catalogue.Manifest{overlay.Name: private, "xterm": parsed, "media": parsedMedia}, Nodes: []compose.Node{ {Name: "anchor", Place: compose.Place{Key: keys["anchor"], Endpoint: "anchor.example:51820", Site: "here", Hub: true, Address: "10.77.0.1"}, Assigned: []string{overlay.Name}, Capabilities: capsSet, Contract: 2, Stated: true}, {Name: "laptop", Place: compose.Place{Key: keys["laptop"], Endpoint: "laptop.example:51820", Site: "here", - Address: "10.77.0.2"}, Assigned: []string{overlay.Name, "xterm"}, Capabilities: capsSet, - Ports: map[string]map[int]int{"xterm": {80: 8080}}, - Settings: map[string][]catalogue.Layer{"xterm": {{From: "laptop", Values: map[string]any{"font": "mono"}}}}}, + Address: "10.77.0.2"}, Assigned: []string{overlay.Name, "xterm", "media"}, Capabilities: capsSet, + Ports: map[string]map[int]int{"xterm": {80: 8080}}, + Settings: map[string][]catalogue.Layer{"xterm": {{From: "laptop", Values: map[string]any{"font": "mono"}}}, + "media": {{From: catalogue.MeshWideLayer, Values: map[string]any{"name": "mesh"}}, + {From: "laptop", Values: map[string]any{"lines": float64(1000)}, Placement: map[string]any{"places": map[string]any{"data": "/srv/data"}}}}}}, }, Holdings: []catalogue.Held{{Claim: "mesh-private-network", Scope: "mesh", Node: "anchor", Module: overlay.Name}}, OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, @@ -142,12 +157,13 @@ func TestInput(t *testing.T) { wj, _ := json.MarshalIndent(want, "", " ") t.Fatalf("Input from the bus differs from the store's:\n got %s\nwant %s", gj, wj) } - if !reflect.DeepEqual(b.valuesRead, []string{"laptop." + overlay.Name, "laptop.xterm"}) { + if !reflect.DeepEqual(b.valuesRead, []string{"laptop." + overlay.Name, "laptop.xterm", "laptop.media"}) { t.Errorf("laptop read config values %v; a node reads its own slice alone", b.valuesRead) } // The same composer over both: the compositions from the bus are the store's, byte for byte. - fps := map[string]string{overlay.Name: "src2:" + strings.Repeat("a", 64), "xterm": "src2:" + strings.Repeat("b", 64)} + fps := map[string]string{overlay.Name: "src2:" + strings.Repeat("a", 64), "xterm": "src2:" + strings.Repeat("b", 64), + "media": "src2:" + strings.Repeat("c", 64)} fromBus, _, err := Compose(ctx, mk(), Mesh{}, Scope{Node: "laptop"}, fps) if err != nil { t.Fatal(err) @@ -180,7 +196,7 @@ func TestInput(t *testing.T) { if every, err := Input(ctx, b, Mesh{}, Scope{Every: true}); err != nil || every.OwnSecrets != nil { t.Fatalf("composing every node: %v, own secrets %v (the vault answers a node about itself alone)", err, every.OwnSecrets) } - if !reflect.DeepEqual(b.valuesRead, []string{"anchor." + overlay.Name, "laptop." + overlay.Name, "laptop.xterm"}) { + if !reflect.DeepEqual(b.valuesRead, []string{"anchor." + overlay.Name, "laptop." + overlay.Name, "laptop.xterm", "laptop.media"}) { // Every node's placed modules, and nothing placed nowhere: anchor places no xterm. t.Errorf("the check read %v", b.valuesRead) } diff --git a/go/delivery/placement.go b/go/delivery/placement.go index 293fb0e..99ed5de 100644 --- a/go/delivery/placement.go +++ b/go/delivery/placement.go @@ -83,14 +83,21 @@ func (p Provider) Chosen() catalogue.Chosen { return catalogue.Chosen{Node: p.No // layers, the mesh's first. Each node's file repeats the mesh-wide layer of the modules placed on it, so a node reads // its own key alone. HasLayer says the node has a layer of its own even when it holds no placement, since a layer // that is there and empty still carries the module's config values (Layers). +// +// MeshValues and Values are the config values of a module in the settings form (its manifest declares no config, so +// mesh-config holds none for it): what a person set in the whole mesh's layer and this node's, beside the placement +// (issue 1126; on 2026-10-26 the media modules on one node, plex's puid and pgid among them). A module in the config +// form takes its values from mesh-config and has none here. type PlacedModule struct { - Module string `json:"module"` - Ports map[string]int `json:"ports,omitempty"` - Bindings map[string]Provider `json:"bindings,omitempty"` - MeshWide map[string]any `json:"mesh-wide,omitempty"` - HasMesh bool `json:"has-mesh-wide,omitempty"` - Layer map[string]any `json:"layer,omitempty"` - HasLayer bool `json:"has-layer,omitempty"` + Module string `json:"module"` + Ports map[string]int `json:"ports,omitempty"` + Bindings map[string]Provider `json:"bindings,omitempty"` + MeshWide map[string]any `json:"mesh-wide,omitempty"` + HasMesh bool `json:"has-mesh-wide,omitempty"` + Layer map[string]any `json:"layer,omitempty"` + HasLayer bool `json:"has-layer,omitempty"` + MeshValues map[string]any `json:"mesh-wide-values,omitempty"` + Values map[string]any `json:"values,omitempty"` } // Holding is one seat a module placed on the node holds (catalogue.Held without its node, which is the file's). @@ -326,11 +333,22 @@ func (p Placement) HostPorts() map[string]map[int]int { // Layers is one module's settings layers on the node, as the composition reads them (compose.Node's Settings): the // whole mesh's layer when it has one, then the node's, which carries the module's config values as mesh-config resolved // them for this node (values, nil when none) beside its placement; a module with no layer and no value has none. This is -// the controller's SettingsFor for a module that takes its config values from mesh-config (issue 664). +// the controller's SettingsFor for a module that takes its config values from mesh-config (issue 664). A module in the +// settings form carries its values in the file (MeshValues, Values), which then stand for mesh-config's. func (m PlacedModule) Layers(node string, values map[string]any) []catalogue.Layer { var out []catalogue.Layer - if m.HasMesh || len(m.MeshWide) > 0 { - out = append(out, catalogue.Layer{From: catalogue.MeshWideLayer, Values: map[string]any{}, Placement: m.MeshWide}) + if m.HasMesh || len(m.MeshWide) > 0 || len(m.MeshValues) > 0 { + mesh := map[string]any{} + for k, v := range m.MeshValues { + mesh[k] = v + } + out = append(out, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh, Placement: m.MeshWide}) + } + if len(m.Values) > 0 { + values = make(map[string]any, len(m.Values)) + for k, v := range m.Values { + values[k] = v + } } if m.HasLayer || len(m.Layer) > 0 || len(values) > 0 { if values == nil { From fe11db9293973f098ef48eebe350a684574d2b39 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:47:17 +0100 Subject: [PATCH 25/42] MergeInto also names programs a commit of the branch added and a later one removed: the merge commit would keep the blob in history (issue 1577) --- go/buildstep/merge.go | 5 +++ go/buildstep/merge_integration_test.go | 30 ++++++++++++++++-- go/buildstep/programs.go | 43 ++++++++++++++++++++++++++ go/buildstep/programs_test.go | 4 +-- 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/go/buildstep/merge.go b/go/buildstep/merge.go index 589c50e..86384b6 100644 --- a/go/buildstep/merge.go +++ b/go/buildstep/merge.go @@ -106,6 +106,11 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm return res, fmt.Errorf("the files the merge adds or modifies cannot be read: %w", err) } res.Programs = programsIn(dir, raw) + history, err := historyPrograms(git, base, commit, res.Programs) + if err != nil { + return res, fmt.Errorf("the commits of the branch cannot be read for compiled programs: %w", err) + } + res.Programs = append(res.Programs, history...) return res, nil } diff --git a/go/buildstep/merge_integration_test.go b/go/buildstep/merge_integration_test.go index ce8518c..f83ece8 100644 --- a/go/buildstep/merge_integration_test.go +++ b/go/buildstep/merge_integration_test.go @@ -79,6 +79,30 @@ func (r *mergeRig) head(files map[string]string) string { return h } +// headThatAddsAndRemoves is head with an earlier commit on the branch that adds a program and a later one that removes it: the +// branch's tree is clean, its history is not. It answers the head and the earlier commit. +func (r *mergeRig) headThatAddsAndRemoves(files map[string]string) (head, added string) { + realGit(r.t, r.work, "checkout", "-q", "-B", "pr", r.fork) + writeFile(r.t, r.work, "gone/prog", "\x7fELF\x02\x01") + realGit(r.t, r.work, "add", ".") + realGit(r.t, r.work, "commit", "-q", "-m", "oops") + added = realGit(r.t, r.work, "rev-parse", "HEAD") + realGit(r.t, r.work, "rm", "-q", "gone/prog") + return r.head2(files), added +} + +// head2 commits files on the current branch and pushes it as the pull request's head. +func (r *mergeRig) head2(files map[string]string) string { + for n, c := range files { + writeFile(r.t, r.work, n, c) + } + realGit(r.t, r.work, "add", ".") + realGit(r.t, r.work, "commit", "-q", "-m", "head") + h := realGit(r.t, r.work, "rev-parse", "HEAD") + realGit(r.t, r.work, "push", "-q", r.bare, "HEAD:refs/pull/1/head") + return h +} + // advance moves the trunk past the fork point with a commit changing files, pushes it and answers its sha. func (r *mergeRig) advance(files map[string]string) string { realGit(r.t, r.work, "checkout", "-q", "-B", "main", r.fork) @@ -102,7 +126,7 @@ func (r *mergeRig) fetch(head string) { // holds both sides' changes. func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing.T) { r := newMergeRig(t) - head := r.head(map[string]string{"b.txt": "head\n", "prog": "\x7fELF\x02\x01"}) + head, added := r.headThatAddsAndRemoves(map[string]string{"b.txt": "head\n", "prog": "\x7fELF\x02\x01"}) base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"}) r.fetch(head) res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote()) @@ -115,8 +139,8 @@ func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing. if !reflect.DeepEqual(res.Changed, []string{"b.txt", "prog"}) { t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt, prog", res.Changed) } - if !reflect.DeepEqual(res.Programs, []Program{{"prog", "ELF"}}) { // issue 1577 - t.Errorf("programs = %v, want the ELF the head adds", res.Programs) + if !reflect.DeepEqual(res.Programs, []Program{{"prog", "ELF", ""}, {"gone/prog", "ELF", added}}) { // issue 1577 + t.Errorf("programs = %v, want the ELF the head adds and the one an earlier commit added and a later one removed", res.Programs) } if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != res.Merge { t.Errorf("the workspace is at %s, not at the merge %s", at, res.Merge) diff --git a/go/buildstep/programs.go b/go/buildstep/programs.go index 8d87dce..36f47e6 100644 --- a/go/buildstep/programs.go +++ b/go/buildstep/programs.go @@ -13,6 +13,9 @@ import ( type Program struct { Path string `json:"path"` Kind string `json:"kind"` // ELF, Mach-O or PE + // Commit is set for a program a commit of the branch added and a later one removed (or replaced): the merge's tree is clean, + // but the merge commit carries the blob into the trunk's history for good. Empty for a program the merge's tree holds. + Commit string `json:"commit,omitempty"` } // ExecutableKind says what executable format the first bytes of a file are, or "" when they are none. A PE needs its header @@ -40,6 +43,46 @@ func ExecutableKind(head []byte) string { // programHeadSize is how many bytes of a file are read to tell its format: enough for a PE header at its usual offsets. const programHeadSize = 0x100 +// historyPrograms names the compiled programs that a commit of base..head added or modified and the merge's tree no longer +// holds at that path (those it holds are in final already): each with the commit that added it. git is the merge's git runner. +func historyPrograms(git func(args ...string) (string, error), base, head string, final []Program) ([]Program, error) { + revs, err := git("rev-list", "--no-merges", "--reverse", base+".."+head) + if err != nil { + return nil, err + } + held := map[string]bool{} + for _, p := range final { + held[p.Path] = true + } + out, seen := []Program{}, map[string]bool{} + for _, c := range strings.Fields(revs) { + raw, err := git("diff-tree", "-r", "-z", "--no-commit-id", "--no-renames", "--root", "--diff-filter=AM", c) + if err != nil { + return nil, err + } + f := strings.Split(raw, "\x00") + for i := 0; i+1 < len(f); i += 2 { + meta := strings.Fields(strings.TrimPrefix(f[i], ":")) + if len(meta) < 4 || (meta[1] != "100644" && meta[1] != "100755") || held[f[i+1]] || seen[meta[3]] { + continue + } + seen[meta[3]] = true + blob, err := git("cat-file", "blob", meta[3]) + if err != nil { + return nil, err + } + head := []byte(blob) + if len(head) > programHeadSize { + head = head[:programHeadSize] + } + if k := ExecutableKind(head); k != "" { + out = append(out, Program{Path: f[i+1], Kind: k, Commit: c}) + } + } + } + return out, nil +} + // programsIn reads the output of `git diff --raw -z --no-renames --diff-filter=AM` (": \0\0" per file) and names each added or modified regular file of the tree dir that is a compiled // program, from its first bytes read locally. A symlink (mode 120000) and a submodule (160000) are not read. It never diff --git a/go/buildstep/programs_test.go b/go/buildstep/programs_test.go index 7cc26bc..e64dae2 100644 --- a/go/buildstep/programs_test.go +++ b/go/buildstep/programs_test.go @@ -60,8 +60,8 @@ func TestProgramsInNamesTheAddedAndModifiedRegularFilesThatArePrograms(t *testin want []Program }{ {"none", "", []Program{}}, - {"a program and a text", line("100755", "m/prog") + line("100644", "m/notes"), []Program{{"m/prog", "ELF"}}}, - {"two programs", line("100644", "m/prog") + line("100755", "m/mac"), []Program{{"m/prog", "ELF"}, {"m/mac", "Mach-O"}}}, + {"a program and a text", line("100755", "m/prog") + line("100644", "m/notes"), []Program{{"m/prog", "ELF", ""}}}, + {"two programs", line("100644", "m/prog") + line("100755", "m/mac"), []Program{{"m/prog", "ELF", ""}, {"m/mac", "Mach-O", ""}}}, {"a symlink and a submodule are not read", line("120000", "m/link") + line("160000", "m/sub"), []Program{}}, {"a path the tree lacks", line("100644", "m/gone"), []Program{}}, } { From 02f50bd9d642870b7b9381c58483de101cdf572e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 19:52:54 +0100 Subject: [PATCH 26/42] A build seat's capacity follows its node's build CPU, and a running ask says its phases Since the build CPU cap (issue 1806) shanks' mesh-build-go kept capacity 3 inside a 12-core slice: three full checks got about 4 cores each and ran past the 30-minute step bound (issue 1890). The puller now judges by min(configured, floor(slice quota cores / cores-per-step)), the quota read live from its cgroup, says it on change and writes capacity. to the pauses state. A run's phases (fetch and merge, digest, tests) are logged with the ask id and written to its running step key; Asker.Wait hands every step value to a WithStepSeen hook. --- go/buildstep/askrun.go | 15 ++++ go/buildstep/capacity_cpu.go | 141 ++++++++++++++++++++++++++++++ go/buildstep/capacity_test.go | 41 ++++++++- go/buildstep/phase.go | 67 ++++++++++++++ go/buildstep/pull.go | 110 ++++++++++++++++++++--- go/buildstep/pull_test.go | 61 ++++++++++++- go/buildstep/seat_tools.go | 7 +- go/delivery/buildrun/asks.go | 17 ++++ go/delivery/buildrun/asks_test.go | 7 +- go/delivery/stepasks.go | 51 +++++++++++ go/delivery/steps.go | 5 ++ 11 files changed, 500 insertions(+), 22 deletions(-) create mode 100644 go/buildstep/capacity_cpu.go create mode 100644 go/buildstep/phase.go diff --git a/go/buildstep/askrun.go b/go/buildstep/askrun.go index d189578..0410f3a 100644 --- a/go/buildstep/askrun.go +++ b/go/buildstep/askrun.go @@ -70,6 +70,11 @@ func (s AskSteps) fetch(ctx context.Context, args map[string]any) (json.RawMessa if s.Fetch == nil { return nil, fetched{}, Refuse("this build module fetches nothing") } + if _, merges := args["base"]; merges { + Phase(ctx, "fetch and merge") + } else { + Phase(ctx, "fetch") + } out, err := s.Fetch(ctx, args) if err != nil { return nil, fetched{}, err @@ -126,6 +131,7 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) if s.Copy == nil { return nil, Refuse("this build module copies no artifact (%s)", a.Artifact) } + Phase(ctx, "copy "+a.Artifact) out, err = s.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From}) } else { if s.Build == nil { @@ -135,6 +141,7 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) if job.Publish { args["publish"] = true } + Phase(ctx, "build "+a.Artifact) out, err = s.Build(ctx, args) } if err != nil { @@ -196,6 +203,13 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string return args } test := func(args map[string]any) (bool, error) { + what := "the whole level" + if dir, _ := args["module"].(string); dir != "" { + what = dir + } else if pkgs, _ := args["packages"].([]string); len(pkgs) > 0 { + what = fmt.Sprintf("%d packages", len(pkgs)) + } + Phase(ctx, fmt.Sprintf("%s: tests of %s", level, what)) out, err := s.Test(ctx, args) if err != nil { return false, err @@ -287,6 +301,7 @@ func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, level string, if job.Run != "" { return nil, true, true } + Phase(ctx, level+": digest") out, err := s.Digest(ctx, withRunner(map[string]any{"workspace": f.Workspace, "level": level})) if err != nil { if s.Logf != nil { diff --git a/go/buildstep/capacity_cpu.go b/go/buildstep/capacity_cpu.go new file mode 100644 index 0000000..05123cc --- /dev/null +++ b/go/buildstep/capacity_cpu.go @@ -0,0 +1,141 @@ +package buildstep + +import ( + "encoding/json" + "fmt" + "math" + "os" + "runtime" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buildcpu" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A build seat's capacity follows the CPU its node gives builds** (issue 1806's follow-up; issue 1890, 2026-10-26 +// 19:41: under the 50 % cap shanks' mesh-build-go kept capacity 3, three full mesh-catalog checks shared 12 cores, and +// each step ran past the 30-minute step bound with no test run). The configured capacity is an upper bound; the node's +// build slice bounds it again by the cores a step needs: floor(the slice's quota in cores / cores-per-step), one at +// least. The quota is read live from the cgroup at every judgement, so a changed cap is followed at the next take +// without a restart, and the result is said (the seat's current verb, the status tools, `capacity.` in the pauses +// state that the delivery view's build_usage reads). + +// CoresPerStepSetting is a build module's setting of the processors one of its steps needs: a number above zero, the +// same on every node (a step's need is the module's, not the node's). Each build module declares it with its own default +// (DefaultCoresPerStep). +const CoresPerStepSetting = "cores-per-step" + +// DefaultCoresPerStep are the build modules' defaults by language, from measured steps (2026-10-26): a full mesh-catalog +// check on mesh-build-go takes what it is given (three at once took 965 % of shanks' 1200 % slice, about 4 cores each, +// and ran past the 30-minute step bound), so a Go step is given 6, half again what starved; a Node step (tsc, esbuild, a +// test runner: median 15 s) and an image build (buildah, its RUN lines mostly fetches) need 2. +var DefaultCoresPerStep = map[string]float64{delivery.AskGo: 6, delivery.AskNode: 2, delivery.AskImage: 2} + +// Quota is the build slice's CPU quota as this process's cgroup holds it, in percent of one processor (1200 is 12 +// cores), and where it was read: the cgroup's cpu.max (buildcpu.ReadInForce, the tighter of its unit's and its slice's), +// else the share the composer gave (buildcpu.ShareWord) over the machine's processors, else 0 and why none is known. +func Quota() (percent int, from string) { + f := buildcpu.ReadInForce() + if f.Quota > 0 { + path := f.Slice.Path + if path == "" { + path = f.Cgroup + } + return f.Quota, "the cpu.max of " + path + } + if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv(buildcpu.ShareWord))); err == nil && n >= 1 && n <= 100 { + return buildcpu.Quota(n, runtime.NumCPU()), fmt.Sprintf("the share %d%% the composer gave (%s), of %d processors; no cpu.max is read", n, buildcpu.ShareWord, runtime.NumCPU()) + } + return 0, "no CPU quota: neither a cpu.max on its cgroup nor a share from the composer" +} + +// EffectiveCapacity is the capacity a seat's holder judges by, from its configured capacity, its node's quota (percent of +// one processor, 0 when none is known) and the cores one of its steps needs. Without a quota it is the configured one; +// with one, min(configured, floor(quota cores / coresPerStep)), and one at least while the configured one is above +// zero. A coresPerStep of zero or less is taken as 1. +func EffectiveCapacity(configured, quotaPercent int, quotaFrom string, coresPerStep float64) delivery.Capacity { + if coresPerStep <= 0 || math.IsNaN(coresPerStep) || math.IsInf(coresPerStep, 0) { + coresPerStep = 1 + } + c := delivery.Capacity{Configured: configured, Effective: configured, QuotaFrom: quotaFrom, CoresPerStep: coresPerStep} + switch { + case configured <= 0: + c.Effective = 0 + c.Why = fmt.Sprintf("its configured capacity is %d, so it pulls nothing", configured) + return c + case quotaPercent <= 0: + c.Why = fmt.Sprintf("its configured capacity %d; %s, so the CPU bounds nothing", configured, quotaFrom) + return c + } + c.QuotaCores = float64(quotaPercent) / 100 + c.ByCPU = int(math.Floor(c.QuotaCores/coresPerStep + 1e-9)) + if c.ByCPU < 1 { + c.ByCPU = 1 + } + byCPU := fmt.Sprintf("%g cores of the build slice (%s) / %g cores a step = %d", c.QuotaCores, quotaFrom, coresPerStep, c.ByCPU) + if c.ByCPU < configured { + c.Effective = c.ByCPU + c.Why = fmt.Sprintf("%d, bounded by the CPU: %s, under its configured %d", c.Effective, byCPU, configured) + } else { + c.Why = fmt.Sprintf("%d, its configured capacity; the CPU allows %s", configured, byCPU) + } + return c +} + +// NodeCoresPerStep is the module's cores-per-step, read from its settings file at path at every call so a change is +// followed at the next judgement; fallback when there is no path, no value or one that is not a number above zero, said +// once per distinct fault through logf. +func NodeCoresPerStep(path string, fallback float64, logf func(string, ...any)) func() float64 { + said := "" + say := func(why string) { + if logf != nil && why != said { + said = why + logf("%s, so a step is taken to need %g cores", why, fallback) + } + } + return func() float64 { + if path == "" { + return fallback + } + raw, err := os.ReadFile(path) + if err != nil { + say(fmt.Sprintf("the setting %s cannot be read: %v", CoresPerStepSetting, err)) + return fallback + } + var s map[string]json.RawMessage + if err := json.Unmarshal(raw, &s); err != nil { + say(fmt.Sprintf("the settings file %s is not JSON: %v", path, err)) + return fallback + } + v, ok := s[CoresPerStepSetting] + if !ok { + return fallback // not declared yet (a module before its manifest names it): the default, nothing to say + } + n, err := coresOf(v) + if err != nil { + say(fmt.Sprintf("the setting %s %s: %v", CoresPerStepSetting, v, err)) + return fallback + } + said = "" + return n + } +} + +// coresOf reads a cores-per-step value: a JSON number or a string holding one, above zero. +func coresOf(v json.RawMessage) (float64, error) { + var n float64 + if err := json.Unmarshal(v, &n); err != nil { + var s string + if err := json.Unmarshal(v, &s); err != nil { + return 0, fmt.Errorf("is neither a number nor a string") + } + if n, err = strconv.ParseFloat(strings.TrimSpace(s), 64); err != nil { + return 0, fmt.Errorf("is not a number") + } + } + if n <= 0 || math.IsNaN(n) || math.IsInf(n, 0) { + return 0, fmt.Errorf("is not above zero") + } + return n, nil +} diff --git a/go/buildstep/capacity_test.go b/go/buildstep/capacity_test.go index a9d1a60..c52f15e 100644 --- a/go/buildstep/capacity_test.go +++ b/go/buildstep/capacity_test.go @@ -37,8 +37,10 @@ func TestNodeCapacity(t *testing.T) { } } -// ADR 0553 §4: a step's threads are the composer's share over the module's own capacity; without the word, none. -func TestNodeLimits(t *testing.T) { +// ADR 0553 §4: a step's threads are the composer's share over the module's own capacity; without the word, none. And +// issue 1806's follow-up: the capacity judged by is the configured one bounded by the build slice's cores over the cores +// a step needs (the three build nodes as they stand on 2026-10-26), and cores-per-step is read from the settings file. +func TestNodeLimitsAndTheCPUBound(t *testing.T) { t.Setenv("MESH_BUILD_CPU_SHARE", "") if l := NodeLimits(3); l.Threads != 0 || l.Slots != 3 { t.Errorf("without the word: %+v", l) @@ -47,4 +49,39 @@ func TestNodeLimits(t *testing.T) { if l := NodeLimits(3); l.Percent != 75 || l.Slots != 3 || l.Threads < 1 { t.Errorf("from the word: %+v", l) } + for _, c := range []struct { + name string + configured, quota int + cores float64 + effective, byCPU int + }{ + {"shanks: 12 cores, configured 3", 3, 1200, 6, 2, 2}, + {"ace: 18 cores, configured 2", 2, 1800, 6, 2, 3}, + {"g14: 8 cores, configured 2", 2, 800, 6, 1, 1}, + {"node on shanks: 12 cores at 2", 3, 1200, 2, 3, 6}, + {"no quota: the configured one", 3, 0, 6, 3, 0}, + {"configured 0 pulls nothing", 0, 1200, 6, 0, 0}, + {"a quota under one step still builds one", 2, 100, 6, 1, 1}, + {"no cores a step is one", 2, 150, 0, 1, 1}, + } { + got := EffectiveCapacity(c.configured, c.quota, "test", c.cores) + if got.Effective != c.effective || got.ByCPU != c.byCPU || got.Why == "" { + t.Errorf("%s: %+v, want effective %d, by CPU %d", c.name, got, c.effective, c.byCPU) + } + } + file := filepath.Join(t.TempDir(), "settings.json") + var said []string + cores := NodeCoresPerStep(file, 6, func(f string, a ...any) { said = append(said, f) }) + for _, c := range []struct { + file string + want float64 + }{{`{"capacity":"shanks=2"}`, 6}, {`{"cores-per-step":4}`, 4}, {`{"cores-per-step":"3.5"}`, 3.5}, {`{"cores-per-step":0}`, 6}, {`{"cores-per-step":0}`, 6}} { + os.WriteFile(file, []byte(c.file), 0o600) + if got := cores(); got != c.want { + t.Errorf("cores-per-step from %s = %g, want %g", c.file, got, c.want) + } + } + if len(said) != 1 { + t.Errorf("a value not above zero is said once, said %d times", len(said)) + } } diff --git a/go/buildstep/phase.go b/go/buildstep/phase.go new file mode 100644 index 0000000..5ca842c --- /dev/null +++ b/go/buildstep/phase.go @@ -0,0 +1,67 @@ +package buildstep + +import ( + "context" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A running ask says its phases** (issue 1890, 2026-10-26 19:09: a mesh-catalog try's step on shanks ran 30 minutes +// and its module's journal held no line naming its ask, so nobody could tell whether it fetched, linted or tested). The +// puller hands every ask's run a phase reporter in its context; RunAsk and a module's own step bodies call Phase as they +// move (fetch and merge, a level's digest, lint, a level's tests), and each phase is logged with the ask's id and its +// time since the start, and written to the ask's running step key (delivery.Step.Phase), so its asker and the delivery +// view read where it stands from the bus. + +type phaseKey struct{} + +// Phase says that the ask whose run ctx belongs to entered phase; nothing outside an ask's run (a tool called by hand). +func Phase(ctx context.Context, phase string) { + if r, ok := ctx.Value(phaseKey{}).(*phases); ok && r != nil { + r.enter(strings.TrimSpace(phase)) + } +} + +// phases is one running ask's reporter: it writes the step key again with each new phase until the step ends. +type phases struct { + mu sync.Mutex + put Put + key string + step delivery.Step + ended bool + logf func(format string, args ...any) + prefix string +} + +func (r *phases) enter(phase string) { + if phase == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.ended || phase == r.step.Phase { + return + } + at := now().UTC() + r.step.Phase, r.step.PhaseAt = phase, at + if r.logf != nil { + r.logf("%s: %s, %s after its start", r.prefix, phase, at.Sub(r.step.Started).Round(time.Second)) + } + if r.put == nil { + return + } + if err := write(r.put, r.key, r.step); err != nil && r.logf != nil { + r.logf("%s: the phase %s could not be written to %s: %v", r.prefix, phase, r.key, err) + } +} + +// end stops the writes (the step's end is written after it) and answers the step with its last phase, for the end. +func (r *phases) end() delivery.Step { + r.mu.Lock() + defer r.mu.Unlock() + r.ended = true + return r.step +} diff --git a/go/buildstep/pull.go b/go/buildstep/pull.go index 960d02e..87b95ee 100644 --- a/go/buildstep/pull.go +++ b/go/buildstep/pull.go @@ -77,6 +77,11 @@ type Puller struct { // for this node, and wakes the puller on each change (watchCapacity): WatchConfigServed in a module, a test's own // double. Until it hands a value the puller judges by Capacity. WatchConfig func(key string, handle func(stdio.ProvisionChange) error) error + // CoresPerStep is what one ask of this module needs, read at every judgement (NodeCoresPerStep); with it the capacity + // judged by is bounded by the node's build slice: min(the configured capacity, floor(the quota's cores / CoresPerStep)) + // (EffectiveCapacity). Nil bounds nothing by the CPU. Quota reads the quota (Quota when nil; a test's own). + CoresPerStep func() float64 + Quota func() (percent int, from string) // Has says whether this node has a need an ask names (delivery.AskNeeds, issue 890). An ask needing what it lacks is // declined: its step ends refused as declined (delivery.StepDeclined), the message is dropped for its asker to put // again, and the worker it came from is not looked at again until the next pull's wait, so a node lacking the need @@ -92,6 +97,11 @@ type Puller struct { // watched is the capacity the config watch handed last, used in place of Capacity once hasWatched. watched int hasWatched bool + // judged is the capacity said last (log and capacity.), so a change is said once. + judged delivery.Capacity + hasJudged bool + capWritten bool + capFault string // declined are the kinds whose worker handed an ask this node declined, left out of the pulls without a wait until // the next pull's wait ends. declined map[string]bool @@ -216,23 +226,85 @@ func (p *Puller) room() bool { // roomNow is room, with the capacity and the asks in hand it was judged on. func (p *Puller) roomNow() (ok bool, capacity, inHand int) { - p.mu.Lock() - watched, has := p.watched, p.hasWatched - p.mu.Unlock() - if has { - capacity = watched - } else if p.Capacity != nil { - capacity = p.Capacity() - } + capacity = p.CapacityDetail().Effective p.mu.Lock() defer p.mu.Unlock() return !p.paused && p.inHand < capacity, capacity, p.inHand } -// CapacityNow is the capacity the puller judges by: the config watch's last value, else Capacity. +// CapacityNow is the capacity the puller judges by (CapacityDetail's Effective). func (p *Puller) CapacityNow() int { - _, capacity, _ := p.roomNow() - return capacity + return p.CapacityDetail().Effective +} + +// CapacityDetail is the capacity the puller judges by now and why: the configured one (the config watch's last value, +// else Capacity), bounded by the node's build slice when CoresPerStep is set (EffectiveCapacity), its quota read now. A +// change from what it said last is logged and written to `capacity.` of the pauses state (PausePut), so the +// delivery view says it too. +func (p *Puller) CapacityDetail() delivery.Capacity { + p.mu.Lock() + watched, has := p.watched, p.hasWatched + p.mu.Unlock() + configured, from := 0, "the settings file" + switch { + case has: + configured, from = watched, "mesh-config, as the config watch handed it" + case p.Capacity != nil: + configured = p.Capacity() + } + var c delivery.Capacity + if p.CoresPerStep == nil { + c = delivery.Capacity{Configured: configured, Effective: configured, QuotaFrom: "not judged: the module names no cores a step needs", + Why: fmt.Sprintf("its configured capacity %d", configured)} + } else { + quota := p.Quota + if quota == nil { + quota = Quota + } + percent, qfrom := quota() + c = EffectiveCapacity(configured, percent, qfrom, p.CoresPerStep()) + } + c.ConfiguredFrom = from + p.sayCapacity(c) + return c +} + +// sayCapacity logs c when it differs from what was said last, and writes it to the node's capacity key until a write +// lands (a failed write is tried again at the next judgement, its error said once). +func (p *Puller) sayCapacity(c delivery.Capacity) { + p.mu.Lock() + same := p.hasJudged && p.judged.Effective == c.Effective && p.judged.Configured == c.Configured && p.judged.ByCPU == c.ByCPU && + p.judged.CoresPerStep == c.CoresPerStep && p.judged.QuotaCores == c.QuotaCores + if !same { + p.judged, p.hasJudged, p.capWritten = c, true, false + } + write := !p.capWritten && p.PausePut != nil + p.mu.Unlock() + if !same { + p.logf("the %s build capacity of %s is %s (configured from %s)", p.Language, p.Node, c.Why, c.ConfiguredFrom) + } + if !write { + return + } + key, err := delivery.CapacityKey(p.Node) + if err != nil { + return + } + c.At = now() + raw, err := delivery.EncodeCapacity(c) + if err == nil { + _, err = p.PausePut(key, json.RawMessage(raw)) + } + p.mu.Lock() + defer p.mu.Unlock() + if err == nil { + p.capWritten, p.capFault = true, "" + return + } + if err.Error() != p.capFault { + p.capFault = err.Error() + p.logf("the %s build capacity of %s could not be written to %s (tried again at the next judgement): %v", p.Language, p.Node, key, err) + } } // Pull takes and runs asks until ctx ends. An ask still running then is neither ended nor acknowledged: the stream @@ -395,6 +467,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { } p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started} p.mu.Unlock() + p.logf("the %s ask %s starts on %s: %s", ask.Kind, ask.ID, p.Node, step.What) go p.run(ctx, msg, ask, key, step) } @@ -425,7 +498,11 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask delivery.StepAsk, key } } }() - result, err := protect(ctx, func(ctx context.Context) (any, error) { return p.Run(ctx, ask) }) + ph := &phases{put: p.Put, key: key, step: step, logf: p.Logf, prefix: fmt.Sprintf("the %s ask %s on %s", ask.Kind, ask.ID, p.Node)} + result, err := protect(ctx, func(ctx context.Context) (any, error) { + return p.Run(context.WithValue(ctx, phaseKey{}, ph), ask) + }) + step = ph.end() // its end keeps the phase it ended in close(done) if ctx.Err() != nil { // The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs. @@ -438,6 +515,15 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask delivery.StepAsk, key _ = msg.Nak() return } + outcome := "passed" + if err != nil { + outcome = "did not pass: " + err.Error() + if len(outcome) > 300 { + outcome = outcome[:300] + "..." + } + } + p.logf("the %s ask %s ended on %s after %s, in its phase %q; %s", ask.Kind, ask.ID, p.Node, + now().Sub(step.Started).Round(time.Second), step.Phase, outcome) if aerr := msg.Ack(); aerr != nil { p.logf("the ask %s ended on %s and its acknowledgement failed: %v", ask.ID, p.Node, aerr) } diff --git a/go/buildstep/pull_test.go b/go/buildstep/pull_test.go index 902a460..fca0690 100644 --- a/go/buildstep/pull_test.go +++ b/go/buildstep/pull_test.go @@ -104,20 +104,38 @@ func TestPullTakesDeliveryBeforeCheck(t *testing.T) { } // 1887/A2a, 1887/AC-pull: given capacity 2 and three asks waiting, two run and the third waits, and it starts when -// one ends, with no timer. +// one ends, with no timer. Issue 1806's follow-up: the 2 is the CPU's, a configured 3 bounded by a 12-core slice at 6 +// cores a step, said once in capacity.; and issue 1890: a run's phase is written to its step key and logged with +// its ask id, and its end keeps the last phase. func TestPullHoldsAtMostItsCapacity(t *testing.T) { src, st := NewFakeAsks(), newSteps() + asks := []delivery.StepAsk{} for i := 0; i < 3; i++ { - _ = src.PublishAsk(testAsk(delivery.AskDelivery)) + a := testAsk(delivery.AskDelivery) + asks = append(asks, a) + _ = src.PublishAsk(a) } release := make(chan struct{}) var mu sync.Mutex started := 0 - p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 2 }, + var said []string + capacities := map[string][]byte{} + quota := 1200 + p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 3 }, + CoresPerStep: func() float64 { return 6 }, + Quota: func() (int, string) { mu.Lock(); defer mu.Unlock(); return quota, "a test's slice" }, + PausePut: func(key string, v any) (uint64, error) { + mu.Lock() + capacities[key] = v.(json.RawMessage) + mu.Unlock() + return 1, nil + }, + Logf: func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }, PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { mu.Lock() started++ mu.Unlock() + Phase(ctx, "unit: tests of 3 packages") <-release return nil, nil }} @@ -129,12 +147,49 @@ func TestPullHoldsAtMostItsCapacity(t *testing.T) { if p.InHand() != 2 || src.Waiting(delivery.AskDelivery) != 1 { t.Fatalf("in hand %d, waiting %d; want 2 and 1", p.InHand(), src.Waiting(delivery.AskDelivery)) } + c := p.CapacityDetail() + if c.Effective != 2 || c.Configured != 3 || c.ByCPU != 2 || c.QuotaCores != 12 || !strings.Contains(c.Why, "bounded by the CPU") { + t.Fatalf("the capacity judged by = %+v, want 2 of a configured 3 bounded by 12 cores / 6", c) + } + mu.Lock() + raw := capacities["capacity.shanks"] + capLines := 0 + for _, l := range said { + if strings.Contains(l, "build capacity of shanks is 2, bounded by the CPU") { + capLines++ + } + } + mu.Unlock() + if got, err := delivery.DecodeCapacity(raw); err != nil || got.Effective != 2 || capLines != 1 { + t.Fatalf("capacity.shanks = %s (%v), said %d times; want effective 2, said once", raw, err, capLines) + } + waitFor(t, "a phase on the step key", func() bool { + key, _ := delivery.AskStepKey("shanks", p.Running()[0].ID) + s, _ := st.get(key) + return s.State == delivery.StepRunning && s.Phase == "unit: tests of 3 packages" && !s.PhaseAt.IsZero() + }) release <- struct{}{} waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 }) // Taken from the source comes a moment before its run starts: wait for the start, not only the take. waitFor(t, "the third started", func() bool { mu.Lock(); defer mu.Unlock(); return started == 3 }) close(release) waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 }) + for _, a := range asks { + key, _ := delivery.AskStepKey("shanks", a.ID) + if s, _ := st.get(key); s.State != delivery.StepPassed || s.Phase != "unit: tests of 3 packages" { + t.Errorf("the end of %s = %+v, want passed in its last phase", a.ID, s) + } + } + mu.Lock() + logged := strings.Join(said, "\n") + quota = 600 // the cap lowered: one step at 6 cores, followed at the next judgement + mu.Unlock() + if !strings.Contains(logged, "the delivery ask "+asks[0].ID+" on shanks: unit: tests of 3 packages") { + t.Errorf("no journal line names the ask's phase: %s", logged) + } + if c := p.CapacityDetail(); c.Effective != 1 { + t.Errorf("after the quota fell to 6 cores, capacity %+v, want 1", c) + } } // An ask that is not one, or not this seat's, is dropped and runs nowhere. And issue 890/AC1: a check ask needing diff --git a/go/buildstep/seat_tools.go b/go/buildstep/seat_tools.go index 07af7eb..e56d54b 100644 --- a/go/buildstep/seat_tools.go +++ b/go/buildstep/seat_tools.go @@ -14,10 +14,13 @@ func SeatTools(seat, what string, p *Puller) []stdio.Tool { Run: func(map[string]any) (any, error) { return run() }} } current := func() (any, error) { - return map[string]any{"node": p.Node, "capacity": p.CapacityNow(), "running": p.Running(), "paused": p.Paused()}, nil + c := p.CapacityDetail() + return map[string]any{"node": p.Node, "capacity": c.Effective, "capacity_why": c.Why, "capacity_detail": c, + "running": p.Running(), "paused": p.Paused()}, nil } return []stdio.Tool{ - verb("current", "The "+what+" asks this node runs now (each one's id, kind and when it started), its build capacity, "+ + verb("current", "The "+what+" asks this node runs now (each one's id, kind and when it started), its build capacity (the configured one, "+ + "bounded by the cores its node's build slice gives over the cores a step needs) and why, "+ "and whether it is paused.", current), verb("pause", "Take no new "+what+" ask on this node until resumed; an ask running here ends in its place. Kept "+ "across a restart (the node's pause key).", func() (any, error) { diff --git a/go/delivery/buildrun/asks.go b/go/delivery/buildrun/asks.go index 59d99d5..a03bbbf 100644 --- a/go/delivery/buildrun/asks.go +++ b/go/delivery/buildrun/asks.go @@ -86,6 +86,9 @@ func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, err if !ok { return delivery.Step{}, errors.New("the watch of the ask's step ended") } + if seen := stepSeenOf(ctx); seen != nil { + seen(step) + } if step.State != delivery.StepRunning { return step, nil } @@ -108,6 +111,20 @@ func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, err } } +type stepSeenKey struct{} + +// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node +// that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so +// (issue 1890). seen runs in the waiting goroutine and must not block. +func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context { + return context.WithValue(ctx, stepSeenKey{}, seen) +} + +func stepSeenOf(ctx context.Context) func(delivery.Step) { + seen, _ := ctx.Value(stepSeenKey{}).(func(delivery.Step)) + return seen +} + // Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason. func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) { if err := k.Put(ctx, a); err != nil { diff --git a/go/delivery/buildrun/asks_test.go b/go/delivery/buildrun/asks_test.go index 0f8540a..6905d2e 100644 --- a/go/delivery/buildrun/asks_test.go +++ b/go/delivery/buildrun/asks_test.go @@ -26,9 +26,10 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)} }} k := Asker{Publisher: f, Waiter: f} - step, err := k.Ask(context.Background(), goAsk("mvl0frv18d1957")) - if err != nil || step.Node != "ace" || len(f.Asks()) != 1 { - t.Fatalf("step %+v, %v, asks %d", step, err, len(f.Asks())) + var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it + step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957")) + if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" { + t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen) } f.Answer = func(a delivery.StepAsk) delivery.Step { return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"} diff --git a/go/delivery/stepasks.go b/go/delivery/stepasks.go index cba0858..33ac861 100644 --- a/go/delivery/stepasks.go +++ b/go/delivery/stepasks.go @@ -693,3 +693,54 @@ func DecodePause(data []byte) (Pause, error) { } return p, nil } + +// CapacityKey is the key, in a build module's PausesState, where the module on node says the capacity it judges by +// now and why: `capacity.` (issue 1806's follow-up). The pauses state has no age, so the value stands until the +// module writes another; a reader such as the delivery view's build_usage says it in place of the configured capacity. +func CapacityKey(node string) (string, error) { + if err := checkName("node", node); err != nil { + return "", err + } + return "capacity." + node, nil +} + +// Capacity is a build seat's capacity on one node as its holder judges by it: the configured capacity (the module's +// setting `capacity`), bounded by the cores its node's build slice gives (its CPU quota, read live) over the cores one +// step needs (the module's setting `cores-per-step`). Effective is min(Configured, ByCPU) when a quota is known, else +// Configured; one step at least while Configured is above zero, so a node whose quota is under one step's need still +// builds, slowly. +type Capacity struct { + V int `json:"v"` + Effective int `json:"effective"` + Configured int `json:"configured"` + // ConfiguredFrom says where Configured was read: mesh-config's value as the config watch handed it, or the settings + // file. + ConfiguredFrom string `json:"configured-from,omitempty"` + // QuotaCores is the build slice's CPU quota in processors (cpu.max: 1200% is 12), 0 when none is known; QuotaFrom + // says where it was read (the cgroup, or the share the composer gave) or why none is known. + QuotaCores float64 `json:"quota-cores,omitempty"` + QuotaFrom string `json:"quota-from"` + // CoresPerStep is what one step of this module needs; ByCPU is floor(QuotaCores / CoresPerStep), at least 1, and 0 + // when no quota is known. + CoresPerStep float64 `json:"cores-per-step"` + ByCPU int `json:"by-cpu,omitempty"` + // Why is the capacity in words: what bounds it. + Why string `json:"why"` + At time.Time `json:"at,omitzero"` +} + +// EncodeCapacity is c as JSON at Version. +func EncodeCapacity(c Capacity) ([]byte, error) { + c.V = Version + c.At = c.At.UTC() + return json.Marshal(c) +} + +// DecodeCapacity reads a capacity value; it refuses an unknown version (ErrUnknownVersion). +func DecodeCapacity(data []byte) (Capacity, error) { + var c Capacity + if err := decode(data, &c); err != nil { + return Capacity{}, err + } + return c, nil +} diff --git a/go/delivery/steps.go b/go/delivery/steps.go index 917278a..095b1f4 100644 --- a/go/delivery/steps.go +++ b/go/delivery/steps.go @@ -73,6 +73,11 @@ type Step struct { // What says what the step is, for a reader of the steps state: an ask's kind, repository, head and levels or // module (issue 1890). Empty for a step a tool started by hand. What string `json:"what,omitempty"` + // Phase is the part of its job a running ask's step is in (fetch, a level's digest, lint or tests), and PhaseAt + // when it entered it: the step key is written again at each phase, so a slow step says where it stands (issue 1890). + // A reader waiting for the end treats such a write as any running one. + Phase string `json:"phase,omitempty"` + PhaseAt time.Time `json:"phase-at,omitzero"` } func (s Step) check() error { From 3968849b1b36a1823628c8324c042ed50d8d3e75 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:08:32 +0100 Subject: [PATCH 27/42] Review of L2: the export is a placement file at its version with its modules (S1); a node asks the vault only when a module placed on it declares an own secret, so one with none composes whether or not the vault answers (H8's failure) (issue 1126) --- go/cmd/placement-seed/main_test.go | 14 ++++++++++++++ go/compose/frombus/export.go | 4 +++- go/compose/frombus/frombus.go | 27 +++++++++++++++++++++++++-- go/compose/frombus/frombus_test.go | 24 +++++++++++++++++++----- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/go/cmd/placement-seed/main_test.go b/go/cmd/placement-seed/main_test.go index 6dd4399..156b4f4 100644 --- a/go/cmd/placement-seed/main_test.go +++ b/go/cmd/placement-seed/main_test.go @@ -1,11 +1,14 @@ package main import ( + "encoding/json" "os" "path/filepath" "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -39,6 +42,17 @@ func TestTheSeedWritesEachNodesPlacementFileOnce(t *testing.T) { if entries, _ := os.ReadDir(filepath.Join(other, "placement")); len(entries) != 0 { t.Errorf("a refused export wrote %v", entries) } + // What the controller's placement-export prints, frombus.Export per node, is what the seed takes: a node with no + // module placed included. + in := compose.Input{Nodes: []compose.Node{{Name: "solo", Place: compose.Place{Hub: true, Address: "10.77.0.9"}}}} + exported, err := frombus.Export(in, "solo") + if err != nil { + t.Fatal(err) + } + answer, _ := json.Marshal(map[string]any{"answer": map[string]delivery.Placement{"solo": exported}}) + if written, err := Seed(strings.NewReader(string(answer)), t.TempDir()); err != nil || len(written) != 1 { + t.Errorf("Export's answer %s was not seeded: %v, %v", answer, written, err) + } for _, bad := range []string{`[]`, `{}`, `{"ace":{"v":1,"node":"novox","modules":[]}}`} { if _, err := Seed(strings.NewReader(bad), t.TempDir()); err == nil { t.Errorf("%s was seeded", bad) diff --git a/go/compose/frombus/export.go b/go/compose/frombus/export.go index ad81a77..ee55f8b 100644 --- a/go/compose/frombus/export.go +++ b/go/compose/frombus/export.go @@ -24,7 +24,9 @@ func Export(in compose.Input, node string) (delivery.Placement, error) { if n == nil { return delivery.Placement{}, fmt.Errorf("frombus: the input has no node %q", node) } - p := delivery.Placement{Node: node, Decided: delivery.NodeDecided{ + // At the version the placement file is read at, with a list of modules even when none is placed: the export is the + // file the seed writes (cmd/placement-seed), and the reader refuses a value of version 0 or without its modules. + p := delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{}, Decided: delivery.NodeDecided{ Endpoint: n.Place.Endpoint, Site: n.Place.Site, Hub: n.Place.Hub, Address: n.Place.Address, PublicDomain: n.PublicDomain, Account: n.Account, AccountHome: n.AccountHome, AgentAccount: n.AgentAccount, AgentHome: n.AgentAccountHome, Adopted: n.Adopted, Taken: n.Taken}} diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 67f2c1e..4601762 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -221,8 +221,12 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). - if in.OwnSecrets, err = r.OwnSecrets(ctx, s.Node); err != nil { - return compose.Input{}, fmt.Errorf("frombus: the own secrets of %s's modules: %w", s.Node, err) + // Asked only when a module placed on it declares one: a node none of whose modules has a secret composes + // whether or not the vault answers. + if declaresOwnSecrets(in, s.Node) { + if in.OwnSecrets, err = r.OwnSecrets(ctx, s.Node); err != nil { + return compose.Input{}, fmt.Errorf("frombus: the own secrets of %s's modules: %w", s.Node, err) + } } // The core's credentials travel as the vault's box sealed to the machine (ADR 0349 §16): the same values, which // the composition reads for a core module as its kept boxes. @@ -257,6 +261,25 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s return compose.CompositionsApplied(in, s.Node, fps) } +// declaresOwnSecrets says a module assigned to node declares an own secret, in the manifest the node composes it from. +func declaresOwnSecrets(in compose.Input, node string) bool { + for _, n := range in.Nodes { + if n.Name != node { + continue + } + for _, module := range n.Assigned { + m, ok := n.Kept[module] + if !ok { + m = in.Catalogue[module] + } + if len(m.OwnSecrets) > 0 { + return true + } + } + } + return false +} + // nodeOf is the composition's node from its record (what the machine observes) and its placement (what a person // decided of it, and where its modules run). func nodeOf(rec delivery.NodeRecord, p delivery.Placement) compose.Node { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index d5ec534..1d7bbcc 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -27,6 +27,7 @@ type bus struct { values map[string]map[string]any // by node.module valuesRead []string own []compose.OwnSecretRow + ownErr error } func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) { @@ -41,7 +42,9 @@ func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { } return raw, nil } -func (b *bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return b.own, nil } +func (b *bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { + return b.own, b.ownErr +} func (b *bus) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { return nil, nil, nil @@ -74,7 +77,7 @@ func TestInput(t *testing.T) { // A module in the settings form (plex and the media modules on one node, 2026-10-26): its config values are in its // layers, not mesh-config, so the placement carries them (MeshValues, Values) and mesh-config's are not read for it. media := []byte(`{"module":"media","version":"1","settings":{"lines":{"kind":"preference","default":0,"why":"kept"},` + - `"name":{"kind":"preference","default":"x","why":"shown"}},"resources":[{"id":"conf","type":"file",` + + `"name":{"kind":"preference","default":"x","why":"shown"}},"own-secrets":{"token":"/srv/media/token"},"resources":[{"id":"conf","type":"file",` + `"path":"/srv/media/conf","mode":"0644","content":"uid=${setting:lines} name=${setting:name}\n"}]}`) mediaDigest := delivery.ManifestDigest(media) keys := map[string]string{"anchor": publicKey(t), "laptop": publicKey(t)} @@ -102,7 +105,7 @@ func TestInput(t *testing.T) { manifests: map[string][]byte{digest: xterm, mediaDigest: media}, values: map[string]map[string]any{"laptop.xterm": {"font": "mono"}, "anchor.xterm": {"font": "serif"}, "laptop.media": {"lines": "not mesh-config's"}}, - own: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, + own: []compose.OwnSecretRow{{Node: "laptop", Module: "media", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, } } @@ -136,9 +139,9 @@ func TestInput(t *testing.T) { {From: "laptop", Values: map[string]any{"lines": float64(1000)}, Placement: map[string]any{"places": map[string]any{"data": "/srv/data"}}}}}}, }, Holdings: []catalogue.Held{{Claim: "mesh-private-network", Scope: "mesh", Node: "anchor", Module: overlay.Name}}, - OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: "xterm", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, + OwnSecrets: []compose.OwnSecretRow{{Node: "laptop", Module: "media", Name: "token", Origin: "made", SealedTo: "sk", Sealed: "{}"}}, // The same value as a core credential's box, for a core module (ADR 0349 §16). - KeptCoreBoxes: []compose.CoreBox{{Node: "laptop", Module: "xterm", Name: "token", SealedTo: "sk", Box: json.RawMessage("{}")}}, + KeptCoreBoxes: []compose.CoreBox{{Node: "laptop", Module: "media", Name: "token", SealedTo: "sk", Box: json.RawMessage("{}")}}, Tunnels: map[string]compose.FoundTunnel{"anchor": {Tunnel: tun}}, Adopted: &compose.AdoptedTunnel{Tunnel: tun, Hub: "anchor", Adopted: true}, // The controller's order is by address; laptop enrolled under its peer's key, and a person named the other. @@ -201,6 +204,17 @@ func TestInput(t *testing.T) { t.Errorf("the check read %v", b.valuesRead) } + // A node none of whose modules declares an own secret composes without asking the vault (the review of L2: a node + // with no secret composed nothing while the vault did not answer); one whose module does, refuses without it. + quiet := mk() + quiet.ownErr = errors.New("no responders") + if _, err := Input(ctx, quiet, Mesh{}, Scope{Node: "anchor"}); err != nil { + t.Errorf("anchor, with no own secret, needed the vault: %v", err) + } + if _, err := Input(ctx, quiet, Mesh{}, Scope{Node: "laptop"}); err == nil { + t.Error("laptop's media, which declares an own secret, composed without the vault") + } + kept := mk() kept.manifests["old"] = xterm in, err := Input(ctx, kept, Mesh{}, Scope{Node: "laptop", Kept: map[string]string{"xterm": "old"}}) From 2c0ab3a592883aaaf5793f0a0b2864f0cb223c24 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:19:54 +0100 Subject: [PATCH 28/42] =?UTF-8?q?A=20have=20carries=20the=20values=20a=20s?= =?UTF-8?q?elf-composing=20node's=20running=20composition=20holds=20sealed?= =?UTF-8?q?=20to=20its=20key,=20so=20its=20composer=20reuses=20them=20afte?= =?UTF-8?q?r=20a=20restart=20(review=20of=20L2,=20round=202;=20issue=20112?= =?UTF-8?q?6,=20ADR=200547=20=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/delivery.go | 13 ++++++++++++- go/delivery/delivery_test.go | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 1966d1b..d4f6288 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -48,7 +48,9 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "regexp" + "slices" "strings" "time" ) @@ -348,6 +350,11 @@ type Have struct { // composition that waits for a person's yes (ADR 0547 §3); empty otherwise. Judgement Judgement `json:"judgement,omitempty"` Waits string `json:"waits,omitempty"` + // Sealed is, for a module the node composes itself, each value its running composition carries sealed to the node's + // key (a secret, a credential, a vault box), as applied (ADR 0547 §3, issue 1126): what the node's composer reuses + // after a restart while it opens to the vault's value, so a value sealed again moves no composition. Openable by the + // node alone, as the vault hands them over the bus. Empty on every other have. + Sealed []string `json:"sealed,omitempty"` } // UserListWritten is the bus's user list as the node-engine of the bus node wrote it to the bus module's file, applied @@ -533,7 +540,11 @@ func DecodePulse(data []byte) (Pulse, error) { func SameHave(a, b Have) bool { ua, ub := a.UserList, b.UserList a.UserList, b.UserList = nil, nil - if a != b || (ua == nil) != (ub == nil) { + if !slices.Equal(a.Sealed, b.Sealed) { + return false + } + a.Sealed, b.Sealed = nil, nil + if !reflect.DeepEqual(a, b) || (ua == nil) != (ub == nil) { return false } if ua == nil { diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index a831685..8c7fc19 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -421,6 +421,17 @@ func TestSame(t *testing.T) { if SameHave(a, b) { t.Error("different haves same") } + // The sealed values a self-composing node runs are part of its have (issue 1126). + sa, sb := a, a + sa.Sealed, sb.Sealed = []string{`{"box":1}`}, []string{`{"box":1}`} + if !SameHave(sa, sb) || SameHave(sa, a) { + t.Error("equal sealed values differ, or one said and one not are the same") + } + if raw, err := EncodeHave(sa); err != nil { + t.Error(err) + } else if back, err := DecodeHave(raw); err != nil || !SameHave(back, sa) { + t.Errorf("sealed values do not travel: %+v, %v", back, err) + } // The user list is compared by what it says, never by where it is held (ADR 0547 §8). at := time.Date(2026, 10, 25, 21, 0, 0, 0, time.UTC) ua, ub := a, a From dd9d9d5b5f01241cef0696be3cb131631ffc7009 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:20:12 +0100 Subject: [PATCH 29/42] Compare haves with SameHave in the round-trip tests, now that a have carries a list (issue 1126) --- go/delivery/delivery_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/delivery/delivery_test.go b/go/delivery/delivery_test.go index 8c7fc19..0ddd2a3 100644 --- a/go/delivery/delivery_test.go +++ b/go/delivery/delivery_test.go @@ -153,7 +153,7 @@ func TestRoundTrip(t *testing.T) { t.Fatal(err) } in.V = Version - if out != in { + if !SameHave(out, in) { t.Fatalf("got %+v want %+v", out, in) } }) @@ -487,7 +487,7 @@ func testHaveRemoved(t *testing.T) { // 1370/AC3 t.Fatal(err) } in.V = Version - if out != in || out.Phase != "removed" { + if !SameHave(out, in) || out.Phase != "removed" { t.Errorf("round trip: %+v", out) } } From 03dce1237803e1640e7d6679089b3fecb84b5d6c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:33:23 +0100 Subject: [PATCH 30/42] The standing test keeps c failing alone after its ask is dropped, so its only recovery is its retirement: clearing every failure at once raced the loop's read of the new list under load (issue 1126; the hidden failure of try-1793042418286545971, issue 1901) --- test/standing.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/standing.test.ts b/test/standing.test.ts index 107cb49..bfc916a 100644 --- a/test/standing.test.ts +++ b/test/standing.test.ts @@ -34,6 +34,8 @@ interface World { ask(list: { as: string; node?: string; values?: Record }[]): Promise; failing: Error | null; holdsErr: Error | null; + /** When set, holds fails only for these consumers (holdsErr's error). */ + holdsErrFor: Set | null; created: string[]; /** Every event but the first success since start, which has its own test. */ said: Said[]; @@ -44,7 +46,7 @@ interface World { async function world(opts: { classify?: boolean; announce?: false; stableForMs?: number } = {}): Promise { const dir = await mkdtemp(join(tmpdir(), "standing-")); const receives = join(dir, "postgres-database.json"); - const w = { dir, failing: null, holdsErr: null, created: [], said: [], all: [] } as unknown as World; + const w = { dir, failing: null, holdsErr: null, holdsErrFor: null, created: [], said: [], all: [] } as unknown as World; w.ask = async (list) => { const given = []; for (const g of list) { @@ -60,8 +62,8 @@ async function world(opts: { classify?: boolean; announce?: false; stableForMs?: }, async remove() {}, async retire() {}, - async holds() { - if (w.holdsErr) throw w.holdsErr; + async holds(p) { + if (w.holdsErr && (!w.holdsErrFor || w.holdsErrFor.has(p.as))) throw w.holdsErr; return true; }, ...(opts.classify ? { classify: () => "its-own" } : {}), @@ -197,9 +199,11 @@ test("a failing consumer the mesh stopped asking for is said recovered when it i await until(() => w.created.length === 3); w.holdsErr = new Error("boom"); await until(() => w.said.filter((s) => s.event === FAILING_EVENT).length >= 3); - // c is no longer asked for while it is still failing; the others then recover. + // c is no longer asked for while it is still failing; the others then recover. c keeps failing, so its only way + // to be said recovered is its retirement: clearing every failure at once raced the loop's read of the new list + // under load (a pass before that read recovered c as an ordinary recovery, with no why). await w.ask([{ as: "a" }, { as: "b" }]); - w.holdsErr = null; + w.holdsErrFor = new Set(["c"]); await until(() => w.said.some((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")); const c = w.said.find((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")!; assert.equal(c.body.why, "retired"); From 43de3c937d49ab8930743058e6ad84311f56e4da Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:44:49 +0100 Subject: [PATCH 31/42] Layouts of the indexer's taken and the view's reach, and the delivery seat's settle verb, so whether a merge is on the machines can be derived (issue 1280, ADR 0556) --- go/catalogue/delivery_seat.go | 13 +++ go/catalogue/verbs_r6_test.go | 2 +- go/delivery/reach.go | 154 ++++++++++++++++++++++++++++++++++ go/delivery/source_test.go | 77 ++++++++++++++++- 4 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 go/delivery/reach.go diff --git a/go/catalogue/delivery_seat.go b/go/catalogue/delivery_seat.go index 9296f5e..82b0803 100644 --- a/go/catalogue/delivery_seat.go +++ b/go/catalogue/delivery_seat.go @@ -75,6 +75,19 @@ func deliveryVerbs() []Verb { Input: schema(map[string]string{"id": "the delivery's id", "why": "the condition it answers"}, []string{"id", "why"})}, deliveryRemovedInR6("retire-history"), + {Name: "settle", Description: "Ask the operator to settle deliveries held because their merge reached the " + + "trunk without a passing check: one ask on the operator's channel for all of them, as an Approve is asked. " + + "The verb moves nothing; on the operator's answer each goes to merged, and to delivered once every node " + + "runs a build containing it (novox/hq ADR 0556). Refused for a delivery not held, or held for another " + + "reason, naming why.", + Input: map[string]any{"type": "object", "additionalProperties": false, "required": []string{"ids", "why"}, + "properties": map[string]any{ + "ids": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "minItems": 1, + "description": "the held deliveries' ids (owner/repository@commit)"}, + "why": map[string]any{"type": "string", "description": "why they may be settled, shown to the operator"}, + }}, + // Added after the seat's release 6 holder shipped: optional until the gate serves it everywhere. + Optional: true}, } } diff --git a/go/catalogue/verbs_r6_test.go b/go/catalogue/verbs_r6_test.go index ddad237..72d8f6f 100644 --- a/go/catalogue/verbs_r6_test.go +++ b/go/catalogue/verbs_r6_test.go @@ -86,7 +86,7 @@ func TestTheDeliverySeatHasNoPlanVerbs(t *testing.T) { if _, ok := rows["mark-ready"]; ok { t.Errorf("mark-ready is still promised") } - for _, kept := range []string{"deliveries", "show", "groups", "checks", "table", "recheck", "close", "stalled"} { + for _, kept := range []string{"deliveries", "show", "groups", "checks", "table", "recheck", "close", "stalled", "settle"} { if v, ok := rows[kept]; !ok || strings.HasPrefix(v.Description, "Removed") { t.Errorf("%s is not kept (ADR 0539 clause 17)", kept) } diff --git a/go/delivery/reach.go b/go/delivery/reach.go new file mode 100644 index 0000000..6562c49 --- /dev/null +++ b/go/delivery/reach.go @@ -0,0 +1,154 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "strings" + "time" +) + +// How far a merge reached (novox/hq ADR 0556, issue 1280). The indexer says, per repository, the last merge it took in +// (TakenBucket); the delivery view derives, per repository, how far each of its modules reached on the nodes that want it +// (ReachBucket); the gate, the one writer of a delivery's record, watches the view's state and moves a merged delivery to +// delivered when every module its merge changed runs, on every node that wants it, at a build containing the merge. + +// TakenBucket is the indexer's state `taken`: one key per repository (RepositoryKey), the last merge it took in. +const TakenBucket = IndexerModule + "_taken" + +// ReachBucket is the delivery view's state `reach`: one key per repository (RepositoryKey), how far its modules reached. +const ReachBucket = ViewModule + "_reach" + +// RepositoryKey is a repository's key in TakenBucket and ReachBucket: its forge name `owner/name` (RepositoryName), in +// lower case as the forge compares it, escaped as a key part (EscapeName). An empty spelling, or one with no owner, is +// refused. +func RepositoryKey(repository string) (string, error) { + name := strings.ToLower(RepositoryName(repository)) + if owner, repo, ok := strings.Cut(name, "/"); !ok || owner == "" || repo == "" { + return "", fmt.Errorf("delivery: %q names no repository as owner/name", repository) + } + return EscapeName(name), nil +} + +// Taken is the value of a taken key: the repository, the merge commit the indexer last took in (applied its message +// whole), and when. +type Taken struct { + V int `json:"v"` + Repository string `json:"repository"` + Merge string `json:"merge"` + At time.Time `json:"at"` +} + +// EncodeTaken is t as JSON at Version; a value with no repository or merge is refused. +func EncodeTaken(t Taken) ([]byte, error) { + if t.Repository == "" || t.Merge == "" { + return nil, fmt.Errorf("delivery: a taken value names its repository and merge") + } + t.V = Version + return json.Marshal(t) +} + +// DecodeTaken reads a taken value; it refuses an unknown version (ErrUnknownVersion) and one with no merge. +func DecodeTaken(data []byte) (Taken, error) { + var t Taken + if err := decode(data, &t); err != nil { + return t, err + } + if t.Repository == "" || t.Merge == "" { + return t, fmt.Errorf("delivery: a taken value names no repository or merge") + } + return t, nil +} + +// Reach is the value of a reach key: how far the modules of one repository reached on the nodes. Indexed is the last +// merge the indexer took in for it (Taken.Merge), empty while it took in none. A repository the indexer holds no module +// of (a library) has no Modules. +type Reach struct { + V int `json:"v"` + Repository string `json:"repository"` + Indexed string `json:"indexed,omitempty"` + Modules map[string]ModuleReach `json:"modules,omitempty"` +} + +// ModuleReach is how far one module reached. FP is its fingerprint at the indexer now; Nodes how many nodes want it +// (a want not removed); Changes its fingerprints the view knows of, each with the merge commit it first appeared at, +// newest first. Floor is the oldest first-appearance commit among the fingerprints its nodes run, set only when every +// node's want is settled (its have equal to the want, running and healthy) and every fingerprint they run is among +// Changes; otherwise Waiting says why there is none. A module no node wants has neither. +type ModuleReach struct { + FP string `json:"fp,omitempty"` + Nodes int `json:"nodes"` + Changes []Change `json:"changes,omitempty"` + Floor string `json:"floor,omitempty"` + Waiting string `json:"waiting,omitempty"` +} + +// Change is one fingerprint of a module and the merge commit it first appeared at (Module.Commit). +type Change struct { + Commit string `json:"commit"` + FP string `json:"fp"` +} + +// MostChanges is how many changes of a module a reach value keeps: as many revisions as the indexer's modules bucket keeps. +const MostChanges = ModulesHistory + +func (r Reach) check() error { + if r.Repository == "" { + return fmt.Errorf("delivery: a reach value names no repository") + } + for name, m := range r.Modules { + if name == "" { + return fmt.Errorf("delivery: a reach value names a module with no name") + } + if m.Nodes < 0 { + return fmt.Errorf("delivery: the reach of %s counts %d nodes", name, m.Nodes) + } + if m.Floor != "" && m.Waiting != "" { + return fmt.Errorf("delivery: the reach of %s has a floor and waits", name) + } + if len(m.Changes) > MostChanges { + return fmt.Errorf("delivery: the reach of %s keeps %d changes, more than %d", name, len(m.Changes), MostChanges) + } + for _, c := range m.Changes { + if c.Commit == "" || c.FP == "" { + return fmt.Errorf("delivery: a change of %s names no commit or fingerprint", name) + } + } + } + return nil +} + +// EncodeReach is r as JSON at Version; a value that does not check is refused. +func EncodeReach(r Reach) ([]byte, error) { + if err := r.check(); err != nil { + return nil, err + } + r.V = Version + return json.Marshal(r) +} + +// DecodeReach reads a reach value; it refuses an unknown version (ErrUnknownVersion) and one that does not check. +func DecodeReach(data []byte) (Reach, error) { + var r Reach + if err := decode(data, &r); err != nil { + return r, err + } + return r, r.check() +} + +// SameReach says whether two reach values are the same, so the view writes only when they differ. +func SameReach(a, b Reach) bool { + if a.Repository != b.Repository || a.Indexed != b.Indexed || len(a.Modules) != len(b.Modules) { + return false + } + for _, name := range slices.Sorted(maps.Keys(a.Modules)) { + x, ok := b.Modules[name] + y := a.Modules[name] + if !ok || x.FP != y.FP || x.Nodes != y.Nodes || x.Floor != y.Floor || x.Waiting != y.Waiting || + !slices.Equal(x.Changes, y.Changes) { + return false + } + } + return true +} diff --git a/go/delivery/source_test.go b/go/delivery/source_test.go index 3eae4bc..0f084ac 100644 --- a/go/delivery/source_test.go +++ b/go/delivery/source_test.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" "testing" + "time" ) func TestSourceKeys(t *testing.T) { // 1200/AC1 @@ -196,7 +197,23 @@ func TestSourceValuesRoundTrip(t *testing.T) { // 1200/AC3 t.Errorf("forge %s round trip: %+v %v", f.Kind, got, err) } } + // 1280: the taken and reach values (ADR 0556). + tk := Taken{Repository: "novox/mesh-sdk", Merge: "cccc", At: time.Date(2026, 10, 26, 20, 0, 0, 0, time.UTC)} + data, err = EncodeTaken(tk) + check("taken", data, err) + tk.V = Version + if got, err := DecodeTaken(data); err != nil || got != tk { + t.Errorf("taken round trip: %+v %v", got, err) + } + r := sampleReach() + data, err = EncodeReach(r) + check("reach", data, err) + if got, err := DecodeReach(data); err != nil || !SameReach(got, r) { + t.Errorf("reach round trip: %+v %v", got, err) + } for name, dec := range map[string]func([]byte) error{ + "taken": func(b []byte) error { _, err := DecodeTaken(b); return err }, + "reach": func(b []byte) error { _, err := DecodeReach(b); return err }, "source": func(b []byte) error { _, err := DecodeSource(b); return err }, "candidate": func(b []byte) error { _, err := DecodeCandidate(b); return err }, "module": func(b []byte) error { _, err := DecodeModule(b); return err }, @@ -282,6 +299,40 @@ func TestSourceValuesRefused(t *testing.T) { // 1200/AC4 wantErr("unknown kind", err, "closed") _, err = EncodeForge(Forge{Kind: ForgePull, Repository: "nope", Number: 1, Head: "h", Action: "opened"}) wantErr("bad repository", err, "nope") + + // 1280: taken and reach values that do not hold (ADR 0556). + _, err = EncodeTaken(Taken{Repository: "a/b"}) + wantErr("taken without merge", err, "merge") + r := sampleReach() + r.Repository = "" + _, err = EncodeReach(r) + wantErr("reach without repository", err, "repository") + r = sampleReach() + m := r.Modules["gitea"] + m.Waiting = "gitea on ace does not yet run src2:b" + r.Modules["gitea"] = m + _, err = EncodeReach(r) + wantErr("reach with a floor that waits", err, "gitea", "floor") + r = sampleReach() + m = r.Modules["gitea"] + m.Changes = append(m.Changes, Change{Commit: "dddd"}) + r.Modules["gitea"] = m + _, err = EncodeReach(r) + wantErr("reach with a change naming no fingerprint", err, "gitea") + r = sampleReach() + m = r.Modules["gitea"] + m.Changes = make([]Change, MostChanges+1) + r.Modules["gitea"] = m + _, err = EncodeReach(r) + wantErr("reach keeping too many changes", err, "more than") +} + +// sampleReach is a reach value of one repository with a module at its floor and a library-like empty one. +func sampleReach() Reach { + return Reach{V: Version, Repository: "novox/mesh-catalog", Indexed: "cccc", Modules: map[string]ModuleReach{ + "gitea": {FP: "src2:a", Nodes: 1, Floor: "cccc", Changes: []Change{{Commit: "cccc", FP: "src2:a"}, {Commit: "bbbb", FP: "src2:b"}}}, + "zsh": {FP: "src2:z", Changes: []Change{{Commit: "aaaa", FP: "src2:z"}}}, + }} } func TestSourceSame(t *testing.T) { // 1200/AC5 @@ -314,14 +365,38 @@ func TestSourceSame(t *testing.T) { // 1200/AC5 if SameForge(f, g) { t.Error("different forge messages are the same") } + if !SameReach(sampleReach(), sampleReach()) { + t.Error("equal reach values differ") + } + for name, change := range map[string]func(*Reach){ + "indexed": func(r *Reach) { r.Indexed = "dddd" }, + "floor": func(r *Reach) { m := r.Modules["gitea"]; m.Floor = "bbbb"; r.Modules["gitea"] = m }, + "changes": func(r *Reach) { m := r.Modules["zsh"]; m.Changes = nil; r.Modules["zsh"] = m }, + "modules": func(r *Reach) { delete(r.Modules, "zsh") }, + } { + r := sampleReach() + change(&r) + if SameReach(r, sampleReach()) { + t.Errorf("reach values differing in %s are the same", name) + } + } } func TestSourceBuckets(t *testing.T) { if SourceBucket != "mesh-delivery-gate_source" || CandidatesBucket != "mesh-delivery-indexer_candidates" || ModulesBucket != "mesh-delivery-indexer_modules" || SourceMainKey != "main" || - SourceHistory != 64 || CandidatesHistory != 10 || ModulesHistory != 20 { + SourceHistory != 64 || CandidatesHistory != 10 || ModulesHistory != 20 || + TakenBucket != "mesh-delivery-indexer_taken" || ReachBucket != "mesh-delivery-view_reach" { t.Error("a bucket or history differs from the decided layout") } + // 1280: a repository keys taken and reach by its forge name, in lower case, from either spelling (ADR 0556). + for in, want := range map[string]string{"novox/Mesh-SDK": "novox/mesh-sdk", "http://h:3000/novox/mesh-sdk.git": "novox/mesh-sdk", + "a/b.c": "a/b=2ec", "": "", "nope": "", "/x": ""} { + got, err := RepositoryKey(in) + if (want == "") != (err != nil) || got != want { + t.Errorf("RepositoryKey(%q) = %q, %v; want %q", in, got, err, want) + } + } } func TestForgeDirs(t *testing.T) { From 54a8afac7356fe409ca4c0da5262c6f77210748d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:48:24 +0100 Subject: [PATCH 32/42] The view's reach carries the reached head and each change's source revision; nothing records a merge as delivered (ADR 0556 v8) --- go/catalogue/delivery_seat.go | 4 ++-- go/delivery/reach.go | 27 ++++++++++++++++++++------- go/delivery/source_test.go | 10 ++++++++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/go/catalogue/delivery_seat.go b/go/catalogue/delivery_seat.go index 82b0803..f1d2d24 100644 --- a/go/catalogue/delivery_seat.go +++ b/go/catalogue/delivery_seat.go @@ -77,8 +77,8 @@ func deliveryVerbs() []Verb { deliveryRemovedInR6("retire-history"), {Name: "settle", Description: "Ask the operator to settle deliveries held because their merge reached the " + "trunk without a passing check: one ask on the operator's channel for all of them, as an Approve is asked. " + - "The verb moves nothing; on the operator's answer each goes to merged, and to delivered once every node " + - "runs a build containing it (novox/hq ADR 0556). Refused for a delivery not held, or held for another " + + "The verb moves nothing; on the operator's answer each goes to merged, and whether it is on the machines " + + "is derived as for any merge (novox/hq ADR 0556). Refused for a delivery not held, or held for another " + "reason, naming why.", Input: map[string]any{"type": "object", "additionalProperties": false, "required": []string{"ids", "why"}, "properties": map[string]any{ diff --git a/go/delivery/reach.go b/go/delivery/reach.go index 6562c49..292bce9 100644 --- a/go/delivery/reach.go +++ b/go/delivery/reach.go @@ -9,10 +9,11 @@ import ( "time" ) -// How far a merge reached (novox/hq ADR 0556, issue 1280). The indexer says, per repository, the last merge it took in -// (TakenBucket); the delivery view derives, per repository, how far each of its modules reached on the nodes that want it -// (ReachBucket); the gate, the one writer of a delivery's record, watches the view's state and moves a merged delivery to -// delivered when every module its merge changed runs, on every node that wants it, at a build containing the merge. +// How far a repository reached (novox/hq ADR 0556, issue 1280). The indexer says, per repository, the last merge it took in +// (TakenBucket); the delivery view keeps, per repository, how far each of its modules reached on the nodes that want it, and +// derives from it the repository's reached head (ReachBucket): the newest head of the repository at which every wanted +// module runs, on every node, a build at or after its last change. A merge at or before the reached head is on the +// machines. Nothing records a merge as delivered (ADR 0429 §1): the gate and the tracker derive it from this state. // TakenBucket is the indexer's state `taken`: one key per repository (RepositoryKey), the last merge it took in. const TakenBucket = IndexerModule + "_taken" @@ -63,12 +64,18 @@ func DecodeTaken(data []byte) (Taken, error) { // Reach is the value of a reach key: how far the modules of one repository reached on the nodes. Indexed is the last // merge the indexer took in for it (Taken.Merge), empty while it took in none. A repository the indexer holds no module -// of (a library) has no Modules. +// of (a library) has no Modules. Reached is the reached head: the newest head of the repository in the source's history, +// not past Indexed, at which every module with a node that wants it has its floor at or after its last change at or before +// that head; ReachedSrc is the source revision that named it. Holds says, in words, what keeps the next head of the +// repository from being reached (a module on a node, the indexer's lag); empty when the repository's newest head is reached. type Reach struct { V int `json:"v"` Repository string `json:"repository"` Indexed string `json:"indexed,omitempty"` Modules map[string]ModuleReach `json:"modules,omitempty"` + Reached string `json:"reached,omitempty"` + ReachedSrc uint64 `json:"reached_src,omitempty"` + Holds []string `json:"holds,omitempty"` } // ModuleReach is how far one module reached. FP is its fingerprint at the indexer now; Nodes how many nodes want it @@ -84,10 +91,12 @@ type ModuleReach struct { Waiting string `json:"waiting,omitempty"` } -// Change is one fingerprint of a module and the merge commit it first appeared at (Module.Commit). +// Change is one fingerprint of a module, the merge commit it first appeared at (Module.Commit), and the source revision +// that named that commit as the repository's head (Module.Src, or the source's history), 0 when not known. type Change struct { Commit string `json:"commit"` FP string `json:"fp"` + Src uint64 `json:"src,omitempty"` } // MostChanges is how many changes of a module a reach value keeps: as many revisions as the indexer's modules bucket keeps. @@ -97,6 +106,9 @@ func (r Reach) check() error { if r.Repository == "" { return fmt.Errorf("delivery: a reach value names no repository") } + if (r.Reached == "") != (r.ReachedSrc == 0) { + return fmt.Errorf("delivery: the reach of %s names its reached head and its source revision together", r.Repository) + } for name, m := range r.Modules { if name == "" { return fmt.Errorf("delivery: a reach value names a module with no name") @@ -139,7 +151,8 @@ func DecodeReach(data []byte) (Reach, error) { // SameReach says whether two reach values are the same, so the view writes only when they differ. func SameReach(a, b Reach) bool { - if a.Repository != b.Repository || a.Indexed != b.Indexed || len(a.Modules) != len(b.Modules) { + if a.Repository != b.Repository || a.Indexed != b.Indexed || len(a.Modules) != len(b.Modules) || + a.Reached != b.Reached || a.ReachedSrc != b.ReachedSrc || !slices.Equal(a.Holds, b.Holds) { return false } for _, name := range slices.Sorted(maps.Keys(a.Modules)) { diff --git a/go/delivery/source_test.go b/go/delivery/source_test.go index 0f084ac..11d3734 100644 --- a/go/delivery/source_test.go +++ b/go/delivery/source_test.go @@ -325,12 +325,16 @@ func TestSourceValuesRefused(t *testing.T) { // 1200/AC4 r.Modules["gitea"] = m _, err = EncodeReach(r) wantErr("reach keeping too many changes", err, "more than") + r = sampleReach() + r.ReachedSrc = 0 + _, err = EncodeReach(r) + wantErr("reach with a head and no revision", err, "reached head") } // sampleReach is a reach value of one repository with a module at its floor and a library-like empty one. func sampleReach() Reach { - return Reach{V: Version, Repository: "novox/mesh-catalog", Indexed: "cccc", Modules: map[string]ModuleReach{ - "gitea": {FP: "src2:a", Nodes: 1, Floor: "cccc", Changes: []Change{{Commit: "cccc", FP: "src2:a"}, {Commit: "bbbb", FP: "src2:b"}}}, + return Reach{V: Version, Repository: "novox/mesh-catalog", Indexed: "cccc", Reached: "cccc", ReachedSrc: 9, Modules: map[string]ModuleReach{ + "gitea": {FP: "src2:a", Nodes: 1, Floor: "cccc", Changes: []Change{{Commit: "cccc", FP: "src2:a", Src: 9}, {Commit: "bbbb", FP: "src2:b", Src: 7}}}, "zsh": {FP: "src2:z", Changes: []Change{{Commit: "aaaa", FP: "src2:z"}}}, }} } @@ -373,6 +377,8 @@ func TestSourceSame(t *testing.T) { // 1200/AC5 "floor": func(r *Reach) { m := r.Modules["gitea"]; m.Floor = "bbbb"; r.Modules["gitea"] = m }, "changes": func(r *Reach) { m := r.Modules["zsh"]; m.Changes = nil; r.Modules["zsh"] = m }, "modules": func(r *Reach) { delete(r.Modules, "zsh") }, + "reached": func(r *Reach) { r.Reached, r.ReachedSrc = "bbbb", 7 }, + "holds": func(r *Reach) { r.Holds = []string{"gitea on ace does not yet run src2:a"} }, } { r := sampleReach() change(&r) From aa8e9afc7c8005175db3a2090ebc1d901b62320d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:52:01 +0100 Subject: [PATCH 33/42] A module's floor counts every running have, healthy or not; health stays in its words (ADR 0556 clause 2, insight) --- go/delivery/reach.go | 11 +++++------ go/delivery/source_test.go | 8 +------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/go/delivery/reach.go b/go/delivery/reach.go index 292bce9..31aeb51 100644 --- a/go/delivery/reach.go +++ b/go/delivery/reach.go @@ -80,9 +80,11 @@ type Reach struct { // ModuleReach is how far one module reached. FP is its fingerprint at the indexer now; Nodes how many nodes want it // (a want not removed); Changes its fingerprints the view knows of, each with the merge commit it first appeared at, -// newest first. Floor is the oldest first-appearance commit among the fingerprints its nodes run, set only when every -// node's want is settled (its have equal to the want, running and healthy) and every fingerprint they run is among -// Changes; otherwise Waiting says why there is none. A module no node wants has neither. +// newest first. Floor is the oldest first-appearance commit among the fingerprints its nodes' haves run, a running have +// counting whether its health check passes or not; it is empty when a node that wants the module has no running have, or +// runs a fingerprint not among Changes. Waiting names, in words, every node where the module is not settled (its have +// different from its want, not running, or not healthy), with or without a floor: health is said here and never moves the +// floor (ADR 0556 clause 2). A module no node wants has neither. type ModuleReach struct { FP string `json:"fp,omitempty"` Nodes int `json:"nodes"` @@ -116,9 +118,6 @@ func (r Reach) check() error { if m.Nodes < 0 { return fmt.Errorf("delivery: the reach of %s counts %d nodes", name, m.Nodes) } - if m.Floor != "" && m.Waiting != "" { - return fmt.Errorf("delivery: the reach of %s has a floor and waits", name) - } if len(m.Changes) > MostChanges { return fmt.Errorf("delivery: the reach of %s keeps %d changes, more than %d", name, len(m.Changes), MostChanges) } diff --git a/go/delivery/source_test.go b/go/delivery/source_test.go index 11d3734..498a8a9 100644 --- a/go/delivery/source_test.go +++ b/go/delivery/source_test.go @@ -309,12 +309,6 @@ func TestSourceValuesRefused(t *testing.T) { // 1200/AC4 wantErr("reach without repository", err, "repository") r = sampleReach() m := r.Modules["gitea"] - m.Waiting = "gitea on ace does not yet run src2:b" - r.Modules["gitea"] = m - _, err = EncodeReach(r) - wantErr("reach with a floor that waits", err, "gitea", "floor") - r = sampleReach() - m = r.Modules["gitea"] m.Changes = append(m.Changes, Change{Commit: "dddd"}) r.Modules["gitea"] = m _, err = EncodeReach(r) @@ -334,7 +328,7 @@ func TestSourceValuesRefused(t *testing.T) { // 1200/AC4 // sampleReach is a reach value of one repository with a module at its floor and a library-like empty one. func sampleReach() Reach { return Reach{V: Version, Repository: "novox/mesh-catalog", Indexed: "cccc", Reached: "cccc", ReachedSrc: 9, Modules: map[string]ModuleReach{ - "gitea": {FP: "src2:a", Nodes: 1, Floor: "cccc", Changes: []Change{{Commit: "cccc", FP: "src2:a", Src: 9}, {Commit: "bbbb", FP: "src2:b", Src: 7}}}, + "gitea": {FP: "src2:a", Nodes: 1, Floor: "cccc", Waiting: "gitea on ace runs src2:a and its health check failed", Changes: []Change{{Commit: "cccc", FP: "src2:a", Src: 9}, {Commit: "bbbb", FP: "src2:b", Src: 7}}}, "zsh": {FP: "src2:z", Changes: []Change{{Commit: "aaaa", FP: "src2:z"}}}, }} } From 5ecda4612493ae58622ab9e08f0a28f11df394f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:57:49 +0100 Subject: [PATCH 34/42] A settle ask's group and the act its options bind, shared by the gate and the teller (ADR 0556 clause 5) --- go/delivery/gate.go | 24 ++++++++++++++++++++++++ go/delivery/gate_test.go | 12 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/go/delivery/gate.go b/go/delivery/gate.go index 5fbe9fb..6a3db12 100644 --- a/go/delivery/gate.go +++ b/go/delivery/gate.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" "regexp" + "sort" "strconv" "strings" @@ -87,6 +88,29 @@ func MergeAskKey(group string, heads []Head) (string, error) { return AskKindMerge + "." + part + "." + hash, nil } +// SettleGroupPrefix begins the group of a settle ask (novox/hq ADR 0556 clause 5): a person's ask to settle deliveries held +// because their merge reached the trunk without a passing check. It is kept in the gate's asks under MergeAskKey like a merge +// ask, so the teller carries it unchanged; its options bind the act AskVerb names for it. +const SettleGroupPrefix = "settle/" + +// SettleGroup is the group of the settle ask of these delivery ids: the prefix and twelve hex digits of the sha256 over the +// ids, sorted and joined by a newline, so the same deliveries asked again keep one key. +func SettleGroup(ids []string) string { + sorted := append([]string(nil), ids...) + sort.Strings(sorted) + sum := sha256.Sum256([]byte(strings.Join(sorted, "\n"))) + return SettleGroupPrefix + hex.EncodeToString(sum[:])[:12] +} + +// AskVerb is the act the options of an ask of the gate bind, by its group: a settle ask's `mesh-delivery.settle`, any other +// `mesh-delivery.merge`. The gate composes the act with it and the teller checks a warrant against it. +func AskVerb(group string) string { + if strings.HasPrefix(group, SettleGroupPrefix) { + return "mesh-delivery.settle" + } + return "mesh-delivery.merge" +} + // ParseAskKey is the inverse of MergeAskKey: the kind, the group and the heads hash. func ParseAskKey(key string) (kind, group, hash string, err error) { parts := strings.Split(key, ".") diff --git a/go/delivery/gate_test.go b/go/delivery/gate_test.go index 71ea691..1378d48 100644 --- a/go/delivery/gate_test.go +++ b/go/delivery/gate_test.go @@ -77,6 +77,18 @@ func TestAskID(t *testing.T) { // 1201/AC2 if AskID(k1) != AskID(k1) { t.Error("the id is not stable") } + // 1280 (ADR 0556 clause 5): a settle ask's group is the same for the same ids in any order, keys like any group, and its + // options bind mesh-delivery.settle; every other ask binds mesh-delivery.merge. + g := SettleGroup([]string{"novox/b@222222222222", "novox/a@111111111111"}) + if g != SettleGroup([]string{"novox/a@111111111111", "novox/b@222222222222"}) || !strings.HasPrefix(g, SettleGroupPrefix) || len(g) != len(SettleGroupPrefix)+12 { + t.Errorf("SettleGroup = %q", g) + } + if k, err := MergeAskKey(g, gateHeads()); err != nil || !asks.UsableID(AskID(k)) { + t.Errorf("the settle ask's key %q: %v", k, err) + } + if AskVerb(g) != "mesh-delivery.settle" || AskVerb("feat/x") != "mesh-delivery.merge" { + t.Errorf("AskVerb = %q, %q", AskVerb(g), AskVerb("feat/x")) + } } func TestOwedKey(t *testing.T) { // 1201/AC3 From 1f70c4c0aa88cb24769519dc5c64ef9d1b313ad6 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:12:24 +0100 Subject: [PATCH 35/42] A settle ask's group begins with a colon, which no branch name holds, so a branch named settle/... still merges on its Approve (review of issue 1280) --- go/delivery/gate.go | 5 +++-- go/delivery/gate_test.go | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go/delivery/gate.go b/go/delivery/gate.go index 6a3db12..450fa5b 100644 --- a/go/delivery/gate.go +++ b/go/delivery/gate.go @@ -90,8 +90,9 @@ func MergeAskKey(group string, heads []Head) (string, error) { // SettleGroupPrefix begins the group of a settle ask (novox/hq ADR 0556 clause 5): a person's ask to settle deliveries held // because their merge reached the trunk without a passing check. It is kept in the gate's asks under MergeAskKey like a merge -// ask, so the teller carries it unchanged; its options bind the act AskVerb names for it. -const SettleGroupPrefix = "settle/" +// ask, so the teller carries it unchanged; its options bind the act AskVerb names for it. Git forbids a colon in a branch +// name, so no pull request's group (its branch) ever begins with it. +const SettleGroupPrefix = "settle:" // SettleGroup is the group of the settle ask of these delivery ids: the prefix and twelve hex digits of the sha256 over the // ids, sorted and joined by a newline, so the same deliveries asked again keep one key. diff --git a/go/delivery/gate_test.go b/go/delivery/gate_test.go index 1378d48..bd4e7bc 100644 --- a/go/delivery/gate_test.go +++ b/go/delivery/gate_test.go @@ -86,8 +86,11 @@ func TestAskID(t *testing.T) { // 1201/AC2 if k, err := MergeAskKey(g, gateHeads()); err != nil || !asks.UsableID(AskID(k)) { t.Errorf("the settle ask's key %q: %v", k, err) } - if AskVerb(g) != "mesh-delivery.settle" || AskVerb("feat/x") != "mesh-delivery.merge" { - t.Errorf("AskVerb = %q, %q", AskVerb(g), AskVerb("feat/x")) + for group, want := range map[string]string{g: "mesh-delivery.settle", "feat/x": "mesh-delivery.merge", + "settle/foo": "mesh-delivery.merge", "settle": "mesh-delivery.merge"} { // a branch named settle/... is a merge's group + if got := AskVerb(group); got != want { + t.Errorf("AskVerb(%q) = %q, want %q", group, got, want) + } } } From e44cbaf57fc34c5d5e88eca664afe32263fb997b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:12:48 +0100 Subject: [PATCH 36/42] A module with no floor holds no head back; a merge among its changes reads not yet on the machines (review of issue 1280, ADR 0556 clause 2) --- go/delivery/reach.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/delivery/reach.go b/go/delivery/reach.go index 31aeb51..ad84483 100644 --- a/go/delivery/reach.go +++ b/go/delivery/reach.go @@ -84,7 +84,8 @@ type Reach struct { // counting whether its health check passes or not; it is empty when a node that wants the module has no running have, or // runs a fingerprint not among Changes. Waiting names, in words, every node where the module is not settled (its have // different from its want, not running, or not healthy), with or without a floor: health is said here and never moves the -// floor (ADR 0556 clause 2). A module no node wants has neither. +// floor (ADR 0556 clause 2). A module whose floor is empty while a node wants it holds no head of the repository back: a +// reader takes a merge among its Changes as not yet on the machines, with these words. A module no node wants has neither. type ModuleReach struct { FP string `json:"fp,omitempty"` Nodes int `json:"nodes"` From d00dac140b0396d691d60c480f2171ae7ee35bc3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:19:16 +0100 Subject: [PATCH 37/42] A build module takes asks only within its node's build hours The operator, 2026-10-26 21:18: g14 builds only from 18:00 to 06:00. A build-hours setting names a window per node in its local time (crossing midnight allowed, absent = any hour); outside it the capacity judged by is 0, running steps finish, and the puller wakes on one timer at the next edge. The change is said in the log and in capacity. with the window and when it opens. --- go/buildstep/buildhours.go | 135 ++++++++++++++++++++++++++++++++++ go/buildstep/capacity_test.go | 50 +++++++++++++ go/buildstep/pull.go | 106 +++++++++++++++++++++++++- go/delivery/stepasks.go | 6 ++ 4 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 go/buildstep/buildhours.go diff --git a/go/buildstep/buildhours.go b/go/buildstep/buildhours.go new file mode 100644 index 0000000..e0bc991 --- /dev/null +++ b/go/buildstep/buildhours.go @@ -0,0 +1,135 @@ +package buildstep + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" +) + +// **A node may build only within its build hours** (the operator, 2026-10-26 21:18: "we should only use g14 as a builder +// outside office hours", "from 18 o'clock until 6 in the morning"). A build module's `build-hours` setting names, per +// node, the window in the node's local time in which it takes asks; outside it the capacity judged by is 0: steps running +// finish, no new one is taken. A node it does not name, or an empty setting, builds at any hour. The puller judges it at +// every judgement and wakes on one timer to the window's next edge (a clock edge, not a poll of another part). + +// BuildHoursSetting is the build modules' setting of the nodes' build windows: `=HH:MM-HH:MM` pairs separated by +// commas, in the node's local time; an end before its start crosses midnight (g14=18:00-06:00). +const BuildHoursSetting = "build-hours" + +// BuildWindow is one node's build hours: minutes after local midnight, Start inclusive, End exclusive. +type BuildWindow struct { + Start, End int +} + +// String is the window as the setting writes it. +func (w BuildWindow) String() string { + return fmt.Sprintf("%02d:%02d-%02d:%02d", w.Start/60, w.Start%60, w.End/60, w.End%60) +} + +// Open says whether t, in its own location, falls in the window. +func (w BuildWindow) Open(t time.Time) bool { + m := t.Hour()*60 + t.Minute() + if w.Start < w.End { + return m >= w.Start && m < w.End + } + return m >= w.Start || m < w.End +} + +// NextEdge is the first time after t, in t's location, at which the window opens or closes. +func (w BuildWindow) NextEdge(t time.Time) time.Time { + var next time.Time + for day := 0; day <= 1; day++ { + for _, m := range []int{w.Start, w.End} { + e := time.Date(t.Year(), t.Month(), t.Day()+day, m/60, m%60, 0, 0, t.Location()) + if e.After(t) && (next.IsZero() || e.Before(next)) { + next = e + } + } + } + return next +} + +// ParseBuildHours is node's window in a build-hours setting: nil when the setting is empty or does not name the node. +func ParseBuildHours(setting, node string) (*BuildWindow, error) { + var found *BuildWindow + for _, pair := range strings.Split(setting, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + name, span, ok := strings.Cut(pair, "=") + from, to, ok2 := strings.Cut(span, "-") + if !ok || !ok2 || strings.TrimSpace(name) == "" { + return nil, fmt.Errorf("the build hours %q: %q is not =HH:MM-HH:MM", setting, pair) + } + start, err := minuteOf(from) + if err != nil { + return nil, fmt.Errorf("the build hours %q: %v", setting, err) + } + end, err := minuteOf(to) + if err != nil { + return nil, fmt.Errorf("the build hours %q: %v", setting, err) + } + if start == end { + return nil, fmt.Errorf("the build hours %q: %s opens and closes at the same minute", setting, pair) + } + if strings.TrimSpace(name) == node { + found = &BuildWindow{Start: start, End: end} + } + } + return found, nil +} + +// minuteOf reads HH:MM as minutes after midnight. +func minuteOf(s string) (int, error) { + var h, m int + s = strings.TrimSpace(s) + if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 { + return 0, fmt.Errorf("%q is not a time HH:MM", s) + } + return h*60 + m, nil +} + +// NodeBuildHours is node's build window, read from the module's settings file at path at every call, and why it is +// none when the value cannot be read. No path, no file value or an empty one is no window and no fault. A value that +// cannot be read is no window too, so the node builds at any hour; the setting is a preference of hours, not a safety +// bound (a node that must never build has capacity 0). It is loud: the fault is said once per distinct fault through +// logf, and the puller writes it in the capacity record that build_usage shows. +func NodeBuildHours(path, node string, logf func(string, ...any)) func() (*BuildWindow, string) { + said := "" + say := func(why string) { + if logf != nil && why != said { + said = why + logf("%s, so %s builds at any hour", why, node) + } + } + fault := func(why string) (*BuildWindow, string) { + say(why) + return nil, why + } + return func() (*BuildWindow, string) { + if path == "" { + return nil, "" + } + raw, err := os.ReadFile(path) + if err != nil { + return fault(fmt.Sprintf("the setting %s cannot be read: %v", BuildHoursSetting, err)) + } + var s map[string]json.RawMessage + if err := json.Unmarshal(raw, &s); err != nil { + return fault(fmt.Sprintf("the settings file %s is not JSON: %v", path, err)) + } + setting := "" + if v, ok := s[BuildHoursSetting]; ok && json.Unmarshal(v, &setting) != nil { + return fault(fmt.Sprintf("the setting %s %s is not a string", BuildHoursSetting, v)) + } + w, err := ParseBuildHours(setting, node) + if err != nil { + return fault(err.Error()) + } + said = "" + return w, "" + } +} diff --git a/go/buildstep/capacity_test.go b/go/buildstep/capacity_test.go index c52f15e..c44bbb5 100644 --- a/go/buildstep/capacity_test.go +++ b/go/buildstep/capacity_test.go @@ -3,7 +3,9 @@ package buildstep import ( "os" "path/filepath" + "strings" "testing" + "time" ) // 1887/A7: the node's build capacity is the build module's setting, read at every pull: the control-node at 0 pulls @@ -84,4 +86,52 @@ func TestNodeLimitsAndTheCPUBound(t *testing.T) { if len(said) != 1 { t.Errorf("a value not above zero is said once, said %d times", len(said)) } + + // The operator, 2026-10-26 21:18: g14 builds only from 18:00 to 06:00, its local time. Outside its build hours the + // capacity judged by is 0, said with the window and when it opens; a node not named, or a bad value, builds at any hour. + cet := time.FixedZone("CET", 3600) + day := func(hh, mm int) time.Time { return time.Date(2026, 10, 26, hh, mm, 0, 0, cet) } + was := now + defer func() { now = was }() + for _, c := range []struct { + name, setting, node string + at time.Time + effective int + edge string // the window's next edge, HH:MM; "" for no window + bad bool + }{ + {"g14 in the office", "g14=18:00-06:00", "g14", day(10, 0), 0, "18:00", false}, + {"g14 in the evening", "g14=18:00-06:00", "g14", day(18, 0), 2, "06:00", false}, + {"g14 past midnight", "g14=18:00-06:00", "g14", day(5, 59), 2, "06:00", false}, + {"g14 at the close", "g14=18:00-06:00", "g14", day(6, 0), 0, "18:00", false}, + {"a day window", "shanks=09:00-17:30,g14=18:00-06:00", "shanks", day(17, 29), 2, "17:30", false}, + {"a node not named", "g14=18:00-06:00", "ace", day(10, 0), 2, "", false}, + {"a bad time", "g14=18:00-6", "g14", day(10, 0), 2, "", true}, + {"no span", "g14=18:00-18:00", "g14", day(10, 0), 2, "", true}, + } { + w, err := ParseBuildHours(c.setting, c.node) + if (err != nil) != c.bad { + t.Errorf("%s: parse %v", c.name, err) + } + if edge := ""; w != nil { + edge = w.NextEdge(c.at).Format("15:04") + if edge != c.edge { + t.Errorf("%s: next edge %s, want %s", c.name, edge, c.edge) + } + } else if c.edge != "" { + t.Errorf("%s: no window, want one", c.name) + } + file := filepath.Join(t.TempDir(), "settings.json") + os.WriteFile(file, []byte(`{"build-hours":"`+c.setting+`"}`), 0o600) + now = func() time.Time { return c.at } + p := &Puller{Language: "go", Node: c.node, Capacity: func() int { return 2 }, BuildHours: NodeBuildHours(file, c.node, nil)} + got := p.CapacityDetail() + if got.Effective != c.effective || got.OutsideHours != (c.effective == 0) || (c.effective == 0 && !strings.Contains(got.Why, "outside its build hours 18:00-06:00 CET until 18:00")) { + t.Errorf("%s: %+v, want capacity %d", c.name, got, c.effective) + } + // A value that cannot be read builds at any hour, loudly: the record names the value and says why. + if c.bad && (!strings.Contains(got.BuildHoursFault, c.setting) || !strings.Contains(got.Why, "at any hour, because its build hours cannot be read")) { + t.Errorf("%s: a bad value is not said in the record: %+v", c.name, got) + } + } } diff --git a/go/buildstep/pull.go b/go/buildstep/pull.go index 87b95ee..893d631 100644 --- a/go/buildstep/pull.go +++ b/go/buildstep/pull.go @@ -82,6 +82,10 @@ type Puller struct { // (EffectiveCapacity). Nil bounds nothing by the CPU. Quota reads the quota (Quota when nil; a test's own). CoresPerStep func() float64 Quota func() (percent int, from string) + // BuildHours is the node's build window, read at every judgement (NodeBuildHours); with WatchConfig set the value + // mesh-config resolves for `build-hours` takes its place once handed. Outside the window the capacity judged by is 0: + // steps running finish, no new one is taken; the puller wakes at the window's next edge. Nil builds at any hour. + BuildHours func() (*BuildWindow, string) // Has says whether this node has a need an ask names (delivery.AskNeeds, issue 890). An ask needing what it lacks is // declined: its step ends refused as declined (delivery.StepDeclined), the message is dropped for its asker to put // again, and the worker it came from is not looked at again until the next pull's wait, so a node lacking the need @@ -97,6 +101,11 @@ type Puller struct { // watched is the capacity the config watch handed last, used in place of Capacity once hasWatched. watched int hasWatched bool + // hours is the build window the config watch handed last, and hoursFault why its value cannot be read; used in place + // of BuildHours once hasHours. + hours *BuildWindow + hoursFault string + hasHours bool // judged is the capacity said last (log and capacity.), so a change is said once. judged delivery.Capacity hasJudged bool @@ -265,16 +274,59 @@ func (p *Puller) CapacityDetail() delivery.Capacity { c = EffectiveCapacity(configured, percent, qfrom, p.CoresPerStep()) } c.ConfiguredFrom = from + w, fault := p.window() + if fault != "" { + c.BuildHoursFault = fault + c.Why = fmt.Sprintf("%s; at any hour, because its build hours cannot be read: %s", c.Why, fault) + } + if w != nil { + at := now() + zone, _ := at.Zone() + c.BuildHours = w.String() + " " + zone + if !w.Open(at) && c.Effective > 0 { + c.OutsideHours = true + c.Why = fmt.Sprintf("0, outside its build hours %s until %s (otherwise %s)", c.BuildHours, w.NextEdge(at).Format("15:04"), c.Why) + c.Effective = 0 + } else if c.Effective > 0 { + c.Why = fmt.Sprintf("%s; within its build hours %s until %s", c.Why, c.BuildHours, w.NextEdge(at).Format("15:04")) + } + } p.sayCapacity(c) return c } +// window is the node's build window now (the config watch's last value, else BuildHours, else none), and why its value +// cannot be read when it cannot. +func (p *Puller) window() (*BuildWindow, string) { + p.mu.Lock() + w, fault, has := p.hours, p.hoursFault, p.hasHours + p.mu.Unlock() + if has { + return w, fault + } + if p.BuildHours != nil { + return p.BuildHours() + } + return nil, "" +} + +// untilEdge is how long until the build window's next edge, and false when the node has no window. +func (p *Puller) untilEdge() (time.Duration, bool) { + w, _ := p.window() + if w == nil { + return 0, false + } + at := now() + return w.NextEdge(at).Sub(at), true +} + // sayCapacity logs c when it differs from what was said last, and writes it to the node's capacity key until a write // lands (a failed write is tried again at the next judgement, its error said once). func (p *Puller) sayCapacity(c delivery.Capacity) { p.mu.Lock() same := p.hasJudged && p.judged.Effective == c.Effective && p.judged.Configured == c.Configured && p.judged.ByCPU == c.ByCPU && - p.judged.CoresPerStep == c.CoresPerStep && p.judged.QuotaCores == c.QuotaCores + p.judged.CoresPerStep == c.CoresPerStep && p.judged.QuotaCores == c.QuotaCores && p.judged.OutsideHours == c.OutsideHours && + p.judged.BuildHours == c.BuildHours && p.judged.BuildHoursFault == c.BuildHoursFault if !same { p.judged, p.hasJudged, p.capWritten = c, true, false } @@ -320,6 +372,7 @@ func (p *Puller) Pull(ctx context.Context) error { p.logf("the pause of %s cannot be read: %v; pulling as not paused", p.Node, err) } go p.watchCapacity() // WatchConfigServed waits for Serve, which a module starts after its pull + go p.watchHours() wait := p.PullWait if wait <= 0 { wait = DefaultPullWait @@ -332,16 +385,29 @@ func (p *Puller) Pull(ctx context.Context) error { // Parked until something wakes it: a capacity change (the config watch), a resume, or a step's end. Every // park logs once, so a puller that sits idle says why (issue 1887). p.logf("the %s puller on %s parks: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused()) + var edge <-chan time.Time // the build window's next edge, when the node has one: one timer, not a poll + stop := func() bool { return false } + if d, ok := p.untilEdge(); ok { + t := time.NewTimer(d + time.Second) + edge, stop = t.C, t.Stop + } select { case <-ctx.Done(): + stop() return ctx.Err() case <-p.wake: + case <-edge: } + stop() _, capacity, inHand = p.roomNow() p.logf("the %s puller on %s wakes: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused()) continue } - msg, err := p.next(ctx, wait) + pullWait := wait + if d, ok := p.untilEdge(); ok && d < pullWait { + pullWait = d // a pull does not wait past the window's close + } + msg, err := p.next(ctx, pullWait) if err != nil { if ctx.Err() != nil { return ctx.Err() @@ -560,6 +626,42 @@ func (p *Puller) watchCapacity() { } } +// watchHours follows the module's config value `build-hours` when WatchConfig and BuildHours are set, as watchCapacity +// follows `capacity`: each value handed is kept, used in place of BuildHours, and wakes the puller. A value that cannot +// be read is no window, said; a watch that cannot start leaves the puller on BuildHours, said once. +func (p *Puller) watchHours() { + if p.WatchConfig == nil || p.BuildHours == nil { + return + } + err := p.WatchConfig(BuildHoursSetting, func(c stdio.ProvisionChange) error { + setting := "" + var w *BuildWindow + var why string + switch { + case c.Refused != "": + why = "its build hours are no longer answered: " + c.Refused + case len(c.Value) > 0 && string(c.Value) != "null" && json.Unmarshal(c.Value, &setting) != nil: + why = fmt.Sprintf("the setting %s %s is not a string", BuildHoursSetting, c.Value) + default: + var err error + if w, err = ParseBuildHours(setting, p.Node); err != nil { + why = err.Error() + } + } + p.mu.Lock() + p.hours, p.hoursFault, p.hasHours = w, why, true + p.mu.Unlock() + if why != "" { + p.logf("the %s puller on %s builds at any hour: %s", p.Language, p.Node, why) + } + p.Wake() + return nil + }) + if err != nil { + p.logf("the %s build hours of %s cannot be followed on the bus, so they are read from the settings file: %v", p.Language, p.Node, err) + } +} + // WatchConfigServed is stdio.WatchConfig once Serve has the runtime's channel (stdio.Served): a build module starts its // pull before it serves, and a watch asked before that would be refused. func WatchConfigServed(key string, handle func(stdio.ProvisionChange) error) error { diff --git a/go/delivery/stepasks.go b/go/delivery/stepasks.go index 33ac861..6b8956a 100644 --- a/go/delivery/stepasks.go +++ b/go/delivery/stepasks.go @@ -724,6 +724,12 @@ type Capacity struct { // when no quota is known. CoresPerStep float64 `json:"cores-per-step"` ByCPU int `json:"by-cpu,omitempty"` + // BuildHours is the node's build window (HH:MM-HH:MM, its local time and zone), absent when it builds at any hour; + // OutsideHours says the time judged fell outside it, so Effective is 0 and the next step waits for the window to open. + BuildHours string `json:"build-hours,omitempty"` + OutsideHours bool `json:"outside-hours,omitempty"` + // BuildHoursFault is why the node's build-hours value cannot be read, naming the value: it then builds at any hour. + BuildHoursFault string `json:"build-hours-fault,omitempty"` // Why is the capacity in words: what bounds it. Why string `json:"why"` At time.Time `json:"at,omitzero"` From 3a095a66bef3e68b6f4605429737384511355944 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 20:33:23 +0100 Subject: [PATCH 38/42] The standing test keeps c failing alone after its ask is dropped, so its only recovery is its retirement: clearing every failure at once raced the loop's read of the new list under load (issue 1126; the hidden failure of try-1793042418286545971, issue 1901) --- test/standing.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/standing.test.ts b/test/standing.test.ts index 107cb49..bfc916a 100644 --- a/test/standing.test.ts +++ b/test/standing.test.ts @@ -34,6 +34,8 @@ interface World { ask(list: { as: string; node?: string; values?: Record }[]): Promise; failing: Error | null; holdsErr: Error | null; + /** When set, holds fails only for these consumers (holdsErr's error). */ + holdsErrFor: Set | null; created: string[]; /** Every event but the first success since start, which has its own test. */ said: Said[]; @@ -44,7 +46,7 @@ interface World { async function world(opts: { classify?: boolean; announce?: false; stableForMs?: number } = {}): Promise { const dir = await mkdtemp(join(tmpdir(), "standing-")); const receives = join(dir, "postgres-database.json"); - const w = { dir, failing: null, holdsErr: null, created: [], said: [], all: [] } as unknown as World; + const w = { dir, failing: null, holdsErr: null, holdsErrFor: null, created: [], said: [], all: [] } as unknown as World; w.ask = async (list) => { const given = []; for (const g of list) { @@ -60,8 +62,8 @@ async function world(opts: { classify?: boolean; announce?: false; stableForMs?: }, async remove() {}, async retire() {}, - async holds() { - if (w.holdsErr) throw w.holdsErr; + async holds(p) { + if (w.holdsErr && (!w.holdsErrFor || w.holdsErrFor.has(p.as))) throw w.holdsErr; return true; }, ...(opts.classify ? { classify: () => "its-own" } : {}), @@ -197,9 +199,11 @@ test("a failing consumer the mesh stopped asking for is said recovered when it i await until(() => w.created.length === 3); w.holdsErr = new Error("boom"); await until(() => w.said.filter((s) => s.event === FAILING_EVENT).length >= 3); - // c is no longer asked for while it is still failing; the others then recover. + // c is no longer asked for while it is still failing; the others then recover. c keeps failing, so its only way + // to be said recovered is its retirement: clearing every failure at once raced the loop's read of the new list + // under load (a pass before that read recovered c as an ordinary recovery, with no why). await w.ask([{ as: "a" }, { as: "b" }]); - w.holdsErr = null; + w.holdsErrFor = new Set(["c"]); await until(() => w.said.some((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")); const c = w.said.find((s) => s.event === RECOVERED_EVENT && s.body.consumer === "c")!; assert.equal(c.body.why, "retired"); From 36ae0c691b713ef730bfdab36727dbe955660fad Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:44:06 +0100 Subject: [PATCH 39/42] The standing and retirement tests wait on what they mean, not on a slept fraction of a bound: announcements are timed from the clock at the moment they are said, short runs are counted in attempts, an addition is proved by a hold it must not wait for (the second timing race of one kind in a day; issue 1126) --- test/retire.test.ts | 14 +++++++++----- test/standing.test.ts | 45 +++++++++++++++++++++++++++---------------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/test/retire.test.ts b/test/retire.test.ts index a000760..25e3155 100644 --- a/test/retire.test.ts +++ b/test/retire.test.ts @@ -109,8 +109,10 @@ test("the fifth pass and not the fourth retires", async () => { // Driven by the clock: five passes 10 ms apart take at least 40 ms after the first sees the set. const w = await world(["a", "b", "c"]); try { - await w.ask(["b", "c"]); + // The clock starts before the file is written: a pass may read it the moment it lands, before any line after the + // write runs, and a start taken after would then count fewer than four intervals. const start = Date.now(); + await w.ask(["b", "c"]); await until(() => w.retired.length === 1); assert.ok(Date.now() - start >= 4 * EVERY - 2, `retired after ${Date.now() - start} ms, before five passes`); } finally { @@ -155,10 +157,10 @@ test("five passes are not enough: the same set must also hold for stableForMs", const HOLD = 400; const w = await world(["a", "b", "c"], { stableForMs: HOLD }); try { + const start = Date.now(); // before the write, as in the fifth-pass test await w.ask(["b", "c"]); - const start = Date.now(); await sleep(EVERY * 10); // well past five passes, well short of the hold - assert.deepEqual(w.retired, [], "retired after five passes, before the set held long enough"); + if (Date.now() - start < HOLD) assert.deepEqual(w.retired, [], "retired after five passes, before the set held long enough"); await until(() => w.retired.length === 1, 3000); assert.ok(Date.now() - start >= HOLD - 5, `retired after ${Date.now() - start} ms, before ${HOLD} ms`); assert.equal((await w.p.retirement()).stable_for_seconds, 0); // 400 ms rounds to 0 s @@ -300,11 +302,13 @@ test("delete refuses an active consumer and deletes only the named retired one", }); test("an addition acts on the first pass", async () => { - const w = await world(["a"]); + // Not gated by the stability a retirement waits for: with a hold of a minute, only a pass that acts at once creates b + // in the time the test waits. A deadline of three intervals, as before, was a race with the scheduler under load. + const w = await world(["a"], { stableForMs: 60_000 }); try { const before = w.created.length; await w.ask(["a", "b"]); - await until(() => w.created.length > before, EVERY * 3); + await until(() => w.created.length > before); assert.equal(w.created.at(-1), "b"); } finally { w.done(); diff --git a/test/standing.test.ts b/test/standing.test.ts index bfc916a..49e3db2 100644 --- a/test/standing.test.ts +++ b/test/standing.test.ts @@ -27,7 +27,7 @@ async function until(cond: () => boolean, ms = 4000): Promise { } } -interface Said { event: string; body: Record } +interface Said { event: string; body: Record; at: number } interface World { dir: string; @@ -37,6 +37,8 @@ interface World { /** When set, holds fails only for these consumers (holdsErr's error). */ holdsErrFor: Set | null; created: string[]; + /** How many times create was asked, failed or not. */ + attempts: number; /** Every event but the first success since start, which has its own test. */ said: Said[]; all: Said[]; @@ -46,7 +48,7 @@ interface World { async function world(opts: { classify?: boolean; announce?: false; stableForMs?: number } = {}): Promise { const dir = await mkdtemp(join(tmpdir(), "standing-")); const receives = join(dir, "postgres-database.json"); - const w = { dir, failing: null, holdsErr: null, holdsErrFor: null, created: [], said: [], all: [] } as unknown as World; + const w = { dir, failing: null, holdsErr: null, holdsErrFor: null, created: [], attempts: 0, said: [], all: [] } as unknown as World; w.ask = async (list) => { const given = []; for (const g of list) { @@ -57,6 +59,7 @@ async function world(opts: { classify?: boolean; announce?: false; stableForMs?: }; const adapter: Adapter = { async create(p) { + w.attempts++; if (w.failing) throw w.failing; w.created.push(p.as); }, @@ -75,8 +78,9 @@ async function world(opts: { classify?: boolean; announce?: false; stableForMs?: failingAfterMs: FAILING_AFTER, sayAgainEveryMs: SAY_AGAIN, stableForMs: opts.stableForMs ?? 0, ...(opts.announce === false ? {} : { announce: (event: string, body: Record) => { - w.all.push({ event, body }); - if (body.why !== "first-success") w.said.push({ event, body }); + const said = { event, body, at: Date.now() }; + w.all.push(said); + if (body.why !== "first-success") w.said.push(said); }, }), }); @@ -94,10 +98,12 @@ test("a consumer failed for minutes is announced, naming it, its machine and the const w = await world(); try { w.failing = new Error(`token request failed: 401 {"error":"invalid_grant"} for pw-a`); + // "Not before failingAfterMs" and "not again too soon" are read from the clock at the moment each is said, not + // from a sleep: a sleep overruns under load and lets the very announcement it checks the absence of fall due. + const asked = Date.now(); await w.ask([{ as: "a", node: "anchor" }]); - await sleep(FAILING_AFTER / 2); - assert.equal(events(w.said), "", "announced before failingAfterMs"); await until(() => w.said.length === 1); + assert.ok(w.said[0].at - asked >= FAILING_AFTER, `announced after ${w.said[0].at - asked} ms, before failingAfterMs`); const b = w.said[0].body; assert.equal(w.said[0].event, FAILING_EVENT); assert.equal(b.consumer, "a"); @@ -105,15 +111,14 @@ test("a consumer failed for minutes is announced, naming it, its machine and the assert.equal(b.class, CLASS_CREDENTIALS); assert.equal(b.provider, "postgres-database"); assert.equal(b["provider-node"], "home-server"); - assert.ok((b.attempts as number) > 3, `attempts ${b.attempts}`); + assert.ok((b.attempts as number) > 1, `attempts ${b.attempts}`); assert.match(String(b.since), /^\d{4}-\d{2}-\d{2}T/); assert.doesNotMatch(String(b.error), /pw-a/, "the consumer's password travelled"); // Said again while it lasts, not every pass. - await sleep(SAY_AGAIN / 2); - assert.equal(events(w.said), FAILING_EVENT, "repeated too soon"); await until(() => w.said.length === 2); assert.equal(events(w.said), `${FAILING_EVENT},${FAILING_EVENT}`); + assert.ok(w.said[1].at - w.said[0].at >= SAY_AGAIN, `repeated after ${w.said[1].at - w.said[0].at} ms, before sayAgainEveryMs`); // The first success takes it back. w.failing = null; @@ -129,15 +134,21 @@ test("a consumer failed for minutes is announced, naming it, its machine and the test("one success between failures starts the run again", async () => { const w = await world(); try { - w.failing = new Error("connection refused"); - await w.ask([{ as: "a" }]); - await sleep(FAILING_AFTER * 0.6); - w.failing = null; + // Each failing run lasts two failed attempts, counted, not a slept fraction of failingAfterMs. A run the load + // stretched to failingAfterMs is a long run and may be announced, so only runs that stayed short are asserted. + const run = async (values?: Record): Promise => { + const start = Date.now(); + const base = w.attempts; + w.failing = new Error("connection refused"); + await w.ask([{ as: "a", values }]); + await until(() => w.attempts >= base + 2); + w.failing = null; + return Date.now() - start; + }; + const first = await run(); await until(() => w.created.length === 1); - w.failing = new Error("connection refused"); - await w.ask([{ as: "a", values: { name: "changed" } }]); - await sleep(FAILING_AFTER * 0.6); - assert.equal(events(w.said), "", "two short runs are not one long one"); + const second = await run({ name: "changed" }); + if (first < FAILING_AFTER && second < FAILING_AFTER) assert.equal(events(w.said), "", "two short runs are not one long one"); } finally { w.stop(); } From cf930388f93bf9f754adafdd4d9225831c73b575 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:49:03 +0100 Subject: [PATCH 40/42] The standing test's runs last 0.6 of failingAfterMs of clock as well as two attempts, so a success that does not reset a run is caught; the addition test holds a thousand stable passes, so 'first pass' is proved (review of the standing test races) --- test/retire.test.ts | 11 ++++++----- test/standing.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/retire.test.ts b/test/retire.test.ts index 25e3155..4b6f4e7 100644 --- a/test/retire.test.ts +++ b/test/retire.test.ts @@ -38,7 +38,7 @@ interface World { async function world( names: string[], - opts: { retire?: boolean; held?: string[]; del?: boolean; reenable?: boolean; stableForMs?: number } = {}, + opts: { retire?: boolean; held?: string[]; del?: boolean; reenable?: boolean; stableForMs?: number; stablePasses?: number } = {}, ): Promise { const dir = await mkdtemp(join(tmpdir(), "retire-")); const receives = join(dir, "postgres-database.json"); @@ -64,6 +64,7 @@ async function world( }; w.p = runProviderLoop("postgres-database", adapter, { receives, everyMs: EVERY, node: "anchor", stableForMs: opts.stableForMs ?? 0, + ...(opts.stablePasses ? { stablePasses: opts.stablePasses } : {}), // Only retirement is these tests'; the standing (provisioner.failing, .recovered) is standing.test.ts's. announce: (event, body) => { if (event === RETIREMENT_EVENT) w.events.push(body); }, }); @@ -302,13 +303,13 @@ test("delete refuses an active consumer and deletes only the named retired one", }); test("an addition acts on the first pass", async () => { - // Not gated by the stability a retirement waits for: with a hold of a minute, only a pass that acts at once creates b - // in the time the test waits. A deadline of three intervals, as before, was a race with the scheduler under load. - const w = await world(["a"], { stableForMs: 60_000 }); + // Not gated by the stability a retirement waits for: with a hold of a minute and a thousand passes, only a pass that + // acts at once creates b. The deadline is a few passes, not three intervals of 10 ms: a scheduler stall must not fail it. + const w = await world(["a"], { stableForMs: 60_000, stablePasses: 1000 }); try { const before = w.created.length; await w.ask(["a", "b"]); - await until(() => w.created.length > before); + await until(() => w.created.length > before, 1000); assert.equal(w.created.at(-1), "b"); } finally { w.done(); diff --git a/test/standing.test.ts b/test/standing.test.ts index 49e3db2..de76be3 100644 --- a/test/standing.test.ts +++ b/test/standing.test.ts @@ -134,14 +134,14 @@ test("a consumer failed for minutes is announced, naming it, its machine and the test("one success between failures starts the run again", async () => { const w = await world(); try { - // Each failing run lasts two failed attempts, counted, not a slept fraction of failingAfterMs. A run the load + // Each failing run lasts two counted failed attempts and at least 0.6 of failingAfterMs of clock (long enough that a success which did not reset the run would be announced at the second run). A run the load // stretched to failingAfterMs is a long run and may be announced, so only runs that stayed short are asserted. const run = async (values?: Record): Promise => { const start = Date.now(); const base = w.attempts; w.failing = new Error("connection refused"); await w.ask([{ as: "a", values }]); - await until(() => w.attempts >= base + 2); + await until(() => w.attempts >= base + 2 && Date.now() - start >= FAILING_AFTER * 0.6); w.failing = null; return Date.now() - start; }; From 18e541c1badca3108249c1c320d2d35bac4c5d82 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 21:55:46 +0100 Subject: [PATCH 41/42] The standing test allows 10 ms between the loop's clock and the test's stamp of an announcement --- test/standing.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/standing.test.ts b/test/standing.test.ts index de76be3..658ec6b 100644 --- a/test/standing.test.ts +++ b/test/standing.test.ts @@ -98,12 +98,13 @@ test("a consumer failed for minutes is announced, naming it, its machine and the const w = await world(); try { w.failing = new Error(`token request failed: 401 {"error":"invalid_grant"} for pw-a`); + // The loop stamps its own clock before it calls announce, so the test's stamp may read a few ms short: 10 ms of slack. // "Not before failingAfterMs" and "not again too soon" are read from the clock at the moment each is said, not // from a sleep: a sleep overruns under load and lets the very announcement it checks the absence of fall due. const asked = Date.now(); await w.ask([{ as: "a", node: "anchor" }]); await until(() => w.said.length === 1); - assert.ok(w.said[0].at - asked >= FAILING_AFTER, `announced after ${w.said[0].at - asked} ms, before failingAfterMs`); + assert.ok(w.said[0].at - asked >= FAILING_AFTER - 10, `announced after ${w.said[0].at - asked} ms, before failingAfterMs`); const b = w.said[0].body; assert.equal(w.said[0].event, FAILING_EVENT); assert.equal(b.consumer, "a"); @@ -118,7 +119,7 @@ test("a consumer failed for minutes is announced, naming it, its machine and the // Said again while it lasts, not every pass. await until(() => w.said.length === 2); assert.equal(events(w.said), `${FAILING_EVENT},${FAILING_EVENT}`); - assert.ok(w.said[1].at - w.said[0].at >= SAY_AGAIN, `repeated after ${w.said[1].at - w.said[0].at} ms, before sayAgainEveryMs`); + assert.ok(w.said[1].at - w.said[0].at >= SAY_AGAIN - 10, `repeated after ${w.said[1].at - w.said[0].at} ms, before sayAgainEveryMs`); // The first success takes it back. w.failing = null; From e78e881ecb3c813852e1369faca8f387df0fd4e7 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 22:38:58 +0100 Subject: [PATCH 42/42] The vault's seat promises own, which L2's vault serves and its claim names, so the controller's catalogue check passes (issue 1126) mesh-catalog's mesh-vault claims its seat serving own (a machine's node-engine asks its own machine's secrets and pair credentials sealed to its stated key), and buscompose already grants that call to node principals (AsksItsOwnSecrets, VaultOwn); only the seat's verb list lacked it, so a claim naming it was refused. --- go/catalogue/seats.go | 10 ++++++++++ go/catalogue/vault_seat_test.go | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 805d0be..312c3b0 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -209,6 +209,16 @@ var defaultSeats = append([]Seat{ "first or changed key is recorded and the machine's core secrets are said changed; the same key " + "again changes nothing.", Input: schema(map[string]string{"key": "the machine's sealing key, a raw X25519 public key in base64"}, nil)}, + // A machine's node-engine composes its own modules and asks their secrets sealed to its key (novox/hq ADR 0547 + // §2, ADR 0550 §5, issue 1126); granted as buscompose's Principal.AsksItsOwnSecrets says. + {Name: "own", Optional: true, Description: "Answer the node-engine of one machine, about its own machine " + + "alone (as the bus proved the caller, never an argument), its modules' own secrets and the pair " + + "credentials it consumes or provides that it names, each sealed to the key it stated with node-key. A " + + "credential the vault does not hold is named missing; never a value in the clear.", + Input: map[string]any{"type": "object", "properties": map[string]any{"credentials": map[string]any{ + "type": "array", "items": map[string]any{"type": "object"}, + "description": "the pair credentials the machine's composition reads, each {role: consumer|provider, " + + "provision, consumer, consumer_module, provider, local}"}}}}, {Name: "move-asks", Optional: true, Description: "Copy rows of the controller's record of asks at a desk " + "into the vault's, as they are and once, from the controller alone. No value is in any of them.", Input: map[string]any{"type": "object", "properties": map[string]any{"asks": map[string]any{ diff --git a/go/catalogue/vault_seat_test.go b/go/catalogue/vault_seat_test.go index ed7c3c1..77630ee 100644 --- a/go/catalogue/vault_seat_test.go +++ b/go/catalogue/vault_seat_test.go @@ -121,6 +121,8 @@ func TestTheVaultsSeatPromisesTheMove(t *testing.T) { "move-in": {"node", "module", "name", "provider", "provision", "consumer", "consumer_module", "local", "origin", "generation", "made_at", "call", "sealed", "read_back_to"}, "move-asks": {"asks"}, + // And own (issue 1126): a machine's node-engine asks its own machine's secrets, sealed to its stated key. + "own": {"credentials"}, } for name, args := range want { var verb *Verb @@ -148,7 +150,7 @@ func TestTheVaultsSeatPromisesTheMove(t *testing.T) { } serving := Manifest{Module: "mesh-vault", Provides: []Offer{{Name: "secret", Scope: ScopeMesh}}, Claims: []Claim{{Name: "mesh-vault", Scope: ScopeMesh, - Serves: []string{"get", "accept", "secret-ask", "issued", "rotate", "move-in", "move-asks"}}}} + Serves: []string{"get", "accept", "secret-ask", "issued", "rotate", "move-in", "node-key", "move-asks", "own"}}}} if err := CanHold(serving, seat); err != nil { t.Fatalf("the vault serving the move was refused: %v", err) }