Design 47's last milestones as one release: the contracts for health, control records, provisioning and history leaving the controller (issues 774, 1126, 1952, 1953, 1954, 1955, 1289) #133

Merged
mesh-tools merged 53 commits from feat/774-l3-reland into main 2026-10-30 11:01:40 +00:00
7 changed files with 291 additions and 7 deletions
Showing only changes of commit 7fdbadc655 - Show all commits
+9 -2
View File
@@ -611,6 +611,10 @@ func healthChecks(t *testing.T) {
bucket("mesh-provisioner", "registry")}},
{Module: "mesh-vault", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}},
{Module: "postgres", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}},
// ADR 0571 clauses 2, 4 (issue 1955): the artifact store's seat says artifacts-gone; the assigner keeps seat-holders.
{Module: "artifact-store-tools", Holds: []Seat{{Name: "artifact-store", Emits: []string{"artifacts-gone"}}},
Reads: []string{delivery.SeatHoldersRead}},
{Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}},
{Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}},
Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}},
{Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools",
@@ -640,8 +644,11 @@ func healthChecks(t *testing.T) {
// Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570
// clauses 1, 8).
"a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"},
"a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"},
"a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"},
"a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak",
"mesh.seat.artifact-store.event.artifacts-gone"},
// Only the assigner writes seat-holders.
"a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"},
"a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"},
// Nor mesh-hand-acts' log and warrants, which it alone writes (ADR 0565 clause 10).
"a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch",
"$KV.mesh-hand-acts_warrants.x", "mesh.log.mesh-hand-acts.acts.x"},
+52
View File
@@ -0,0 +1,52 @@
package catalogue
// **The artifact store's sweep, out of the controller** (novox/hq ADR 0571 clause 2, issue 1955): the seat
// `artifact-store`, held once for the mesh by artifact-store-tools, serves what the controller served of its store
// (artifacts, held, mirrors, images, collect) and says `artifacts-gone`. Its keep set is the store's own listing joined
// with every want's fingerprint and each module's five newest builds from the builder's log (ADR 0526 §1); it lets go
// of nothing until the controller's lease names a holder that is no controller.
// ArtifactStoreSeat is the seat artifact-store-tools holds.
const ArtifactStoreSeat = "artifact-store"
// artifactStoreVerbs are the seat's verbs, with the controller's arguments so a caller changes only the seat it asks.
func artifactStoreVerbs() []Verb {
return []Verb{
{Name: "artifacts", Description: "Every artifact the mesh made that lives in its artifact store, each with its " +
"repository, digest, kind (image or archive) and state: kept, and why (a want names it, or one of the five " +
"newest builds of its module in the builder's log), eligible (made, kept for no reason, not yet let go), or " +
"unrecorded (in the store and named by nothing the mesh made: never removed). Reads only.",
Input: schema(map[string]string{
"repository": "only this repository's references, as <module>/<artifact> (the counts stay whole)",
"collected": "\"true\": list the collected references too (counted always)",
}, nil, "collected")},
{Name: "held", Description: "Whether the artifact store holds each reference given, asked with one HEAD each: " +
"held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key. " +
"Reads only.",
Input: listed(schema(map[string]string{
"references": "the artifacts' references, artifact-store://<module>/<artifact>/blobs/sha256:<hex>: a list, " +
"or one text separated by commas",
}, []string{"references"}), "references")},
{Name: "collect", Description: "The artifact store's sweep, now: what is eligible is let go of. Without confirm a " +
"dry run: what would be let go, nothing held or deleted. With confirm and why: every kept archive held first, " +
"then each eligible artifact let go of, oldest first, an index with the platform manifests no kept index " +
"names, each deletion recorded in the hand-act log. Bounded by most (500 by default, at most 5000) and 45 " +
"seconds; what is left is said. Never touches a digest the mesh did not make.",
Input: schema(map[string]string{
"confirm": "\"true\": let go of what is eligible (needs why); without it nothing is done",
"why": "with confirm: why, required and recorded",
"most": "let go of at most this many (500 by default, at most 5000)",
}, nil, "confirm")},
{Name: "mirrors", Description: "The bases the mesh copied into its artifact store for builds to stand on, each " +
"with its state: kept because a kept build stood on it, or eligible. With record and confirm and why, copies " +
"no record names are recorded as the mesh's. Deletes nothing.",
Input: schema(map[string]string{
"record": "references to record, artifact-store://<repository>@sha256:<hex>, separated by spaces or commas",
"confirm": "\"true\": record them (needs why); without it nothing is recorded",
"why": "with confirm: why, required and recorded",
}, nil, "confirm")},
{Name: "images", Description: "Every container image one machine's want and have name, each with the resources " +
"naming it: what the machine keeps when it prunes its images. Reads only.",
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
}
}
+4 -1
View File
@@ -39,7 +39,10 @@ func placementVerbs() []Verb {
"history, one node's or one module's changes in order: a revision the state keeps whole, an older one as the " +
"proposal's changes from its placement-changed event.",
Input: schema(map[string]string{"node": "one node", "module": "one module: the nodes it is placed on",
"history": "\"true\": the changes in order, from the stream and the state's revisions"}, nil, "history")},
"history": "\"true\": the changes in order, from the stream and the state's revisions",
// ADR 0571 clause 4: who holds which seat on which node, from the state seat-holders.
"seats": "\"true\": who holds each seat, and on which node (the state seat-holders); with node, that node's"},
nil, "history", "seats")},
{Name: PlacementVerbProposals, Optional: true, Description: "Every open proposal of placement: its changes, " +
"the revision of each key it was laid over, its candidates, every node's answer, the differences computed and " +
"answered, its verdict and its ask. Narrowed to one by id.",
+3
View File
@@ -303,6 +303,9 @@ var defaultSeats = append([]Seat{
// policy and the bus step at the assigner, and the hand-act log.
{Name: WantsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"policy-changed"}, Serves: wantsVerbs()},
{Name: HandActsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"act-recorded"}, Serves: handActsVerbs()},
// The artifact store's sweep (novox/hq ADR 0571 clause 2, artifact_store_seat.go).
{Name: ArtifactStoreSeat, Scope: ScopeMesh, Decision: "novox/hq ADR 0571", Emits: []string{"artifacts-gone"},
Serves: artifactStoreVerbs()},
// Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a
// delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight.
// They keep their names until that migration is done deliberately, apart from the node-* pass.
+6 -4
View File
@@ -76,9 +76,11 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
// And ADR 0565 clause 1's two.
{WantsSeat, "upgrade,bus", "policy-changed"},
{HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"},
// ADR 0571 clause 2: the artifact store's sweep.
{ArtifactStoreSeat, "artifacts,held,collect,mirrors,images", "artifacts-gone"},
} {
s, ok := SeatNamed(c.seat)
if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 056") ||
if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 05") ||
strings.Join(VerbNames(s.Serves), ",") != c.verbs || strings.Join(s.Emits, ",") != c.emits {
t.Errorf("the health seat %s is %v, %+v", c.seat, ok, s)
}
@@ -120,9 +122,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
// mesh-build-machine row goes, when no registered manifest claims it.
// Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1);
// fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts
// (novox/hq ADR 0565 clause 1).
if len(Seats()) != 55 {
t.Errorf("the mesh defines %d seats rather than 55; the set is closed, so a change here is "+
// (novox/hq ADR 0565 clause 1); fifty-six with artifact-store (novox/hq ADR 0571 clause 2).
if len(Seats()) != 56 {
t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
}
}
+38
View File
@@ -56,6 +56,44 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) {
if _, err := EncodeNodeHealth(NodeHealth{}); err == nil {
t.Error("a report naming no node was encoded")
}
// ADR 0571 clause 4: who holds which seat, from placement; a node seat by its node.
held := SeatHoldersOf(map[string]Placement{
"ace": {Node: "ace", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}},
"novox": {Node: "novox", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"}}},
})
keys := map[string][]byte{}
for k, v := range held {
keys[k], _ = json.Marshal(v)
}
if h, err := HoldersOf(keys, "mesh-dns-resolver", ""); err != nil || len(h) != 2 || h[0].Node != "ace" {
t.Errorf("a replicated seat's holders are %v, %v", h, err)
}
if h, _ := HoldersOf(keys, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 3 {
t.Errorf("a node seat's holder is %v, keys %v", h, held)
}
if h, _ := HoldersOf(keys, "node-files", "novox"); h != nil {
t.Errorf("a node seat nobody holds there is held by %v", h)
}
// ADR 0571 clause 1: the builder's log, a start and an end, and the duration between them.
at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Node: "ace", At: at}
end := start
end.Event, end.At, end.Outcome, end.Artifacts = BuildEndedEvent, at.Add(3*time.Minute), "done", []BuiltDigest{{Name: "image", Digest: "sha256:1"}}
for _, e := range []BuildLogEntry{start, end} {
raw, err := EncodeBuildLogEntry(e)
back, derr := DecodeBuildLogEntry(raw)
if err != nil || derr != nil || back.Event != e.Event || back.Outcome != e.Outcome {
t.Errorf("%s read back as %+v, %v %v", e.Event, back, err, derr)
}
}
if d, ok := BuildDuration(start, end); !ok || d != 3*time.Minute {
t.Errorf("the build took %s, %v", d, ok)
}
bad := end
bad.Outcome = ""
if _, err := EncodeBuildLogEntry(bad); err == nil {
t.Error("an end with no outcome was encoded")
}
if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil {
t.Errorf("a held value's key is %q, %v", k, err)
}
+179
View File
@@ -0,0 +1,179 @@
package delivery
import (
"encoding/json"
"fmt"
"sort"
"time"
)
// **Who holds which seat, and the builder's log** (novox/hq ADR 0571 clauses 1 and 4, issue 1955).
// SeatHoldersState is the state the seat mesh-placement's holder writes of who holds which seat on which node (clause 4):
// one key per seat held once for the mesh, `<seat>`, and one per node of a seat held on every node, `<seat>.<node>`.
const (
SeatHoldersState = "seat-holders"
SeatHoldersBucket = AssignerModule + "_" + SeatHoldersState
SeatHoldersRead = AssignerModule + "." + SeatHoldersState
)
// SeatHolder is one module holding a seat on a node.
type SeatHolder struct {
Module string `json:"module"`
Node string `json:"node"`
}
// SeatHolders is one key of seat-holders: the seat, its scope, the node for a node seat's key, and its holders
// (several only for a replicated mesh seat), sorted.
type SeatHolders struct {
V int `json:"v"`
Seat string `json:"seat"`
Scope string `json:"scope"`
Node string `json:"node,omitempty"`
Holders []SeatHolder `json:"holders"`
}
// SeatHoldersKey is a seat's key: `<seat>` for a seat held once for the mesh (node empty), `<seat>.<node>` for a node
// seat on one node.
func SeatHoldersKey(seat, node string) (string, error) {
if err := checkName("seat", seat); err != nil {
return "", err
}
if node == "" {
return seat, nil
}
if err := checkName("node", node); err != nil {
return "", err
}
return seat + "." + node, nil
}
// SeatHoldersOf is the state composed from every node's placement (ADR 0567 §1): each holding by the seat it claims,
// keyed as SeatHoldersKey says, with the module holding it and its node. Deterministic, so a writer writes only a key
// that differs.
func SeatHoldersOf(placements map[string]Placement) map[string]SeatHolders {
out := map[string]SeatHolders{}
nodes := make([]string, 0, len(placements))
for n := range placements {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
for _, h := range placements[n].Holds {
node := ""
if h.Scope == "node" {
node = n
}
key, err := SeatHoldersKey(h.Claim, node)
if err != nil {
continue
}
sh := out[key]
sh.V, sh.Seat, sh.Scope, sh.Node = Version, h.Claim, h.Scope, node
sh.Holders = append(sh.Holders, SeatHolder{Module: h.Module, Node: n})
out[key] = sh
}
}
for k, sh := range out {
sort.Slice(sh.Holders, func(i, j int) bool {
return sh.Holders[i].Node+"/"+sh.Holders[i].Module < sh.Holders[j].Node+"/"+sh.Holders[j].Module
})
out[k] = sh
}
return out
}
// HoldersOf reads who holds seat (on node, for a node seat) from the state's keys as a reader watched them: the node's
// key first, then the seat's. None when nobody holds it.
func HoldersOf(keys map[string][]byte, seat, node string) ([]SeatHolder, error) {
for _, n := range []string{node, ""} {
key, err := SeatHoldersKey(seat, n)
if err != nil {
if n == "" {
return nil, err
}
continue
}
raw, ok := keys[key]
if !ok {
continue
}
var sh SeatHolders
if err := json.Unmarshal(raw, &sh); err != nil {
return nil, fmt.Errorf("delivery: the holders of %s are not readable: %w", key, err)
}
if sh.V != Version {
return nil, fmt.Errorf("delivery: the holders of %s are of version %d, and this reads %d", key, sh.V, Version)
}
return sh.Holders, nil
}
return nil, nil
}
// The builder's log (clause 1): a stream of its own, `log`, one entry when a build of a fingerprint starts and one when
// it ends, kept 30 days.
const (
BuilderLogStream = "log"
BuildStartedEvent = "build.started"
BuildEndedEvent = "build.ended"
// BuilderLogDays is how long the log keeps its entries.
BuilderLogDays = 30
)
// BuiltDigest is one artifact a build ended with, by its name and digest.
type BuiltDigest struct {
Name string `json:"name"`
Digest string `json:"digest"`
}
// BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, the build node and
// the time; at the end, the outcome and each artifact's digest.
type BuildLogEntry struct {
V int `json:"v"`
Event string `json:"event"`
Module string `json:"module"`
FP string `json:"fp"`
Source string `json:"source"`
Node string `json:"node"`
At time.Time `json:"at"`
Outcome string `json:"outcome,omitempty"`
Artifacts []BuiltDigest `json:"artifacts,omitempty"`
}
// EncodeBuildLogEntry is e as JSON at Version; it refuses an entry a reader could not place.
func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) {
e.V = Version
switch {
case e.Event != BuildStartedEvent && e.Event != BuildEndedEvent:
return nil, fmt.Errorf("delivery: a build log entry is %s or %s, not %q", BuildStartedEvent, BuildEndedEvent, e.Event)
case e.Module == "" || e.FP == "" || e.Node == "" || e.At.IsZero():
return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, node and time")
case e.Event == BuildEndedEvent && e.Outcome == "":
return nil, fmt.Errorf("delivery: a build's end names its outcome")
case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Artifacts) > 0):
return nil, fmt.Errorf("delivery: a build's start names no outcome and no artifact")
}
return json.Marshal(e)
}
// DecodeBuildLogEntry reads one entry.
func DecodeBuildLogEntry(data []byte) (BuildLogEntry, error) {
var e BuildLogEntry
if err := json.Unmarshal(data, &e); err != nil {
return BuildLogEntry{}, fmt.Errorf("delivery: a build log entry is not readable: %w", err)
}
if e.V != Version {
return BuildLogEntry{}, fmt.Errorf("delivery: a build log entry of version %d, and this reads %d", e.V, Version)
}
return e, nil
}
// BuildDuration is a build's duration, its end less its start (clause 1): false unless both are the same module's
// build of the same fingerprint, the start first.
func BuildDuration(start, end BuildLogEntry) (time.Duration, bool) {
if start.Event != BuildStartedEvent || end.Event != BuildEndedEvent || start.Module != end.Module || start.FP != end.FP ||
end.At.Before(start.At) {
return 0, false
}
return end.At.Sub(start.At), true
}