Design 47's last milestones as one release: the contracts for health, control records, provisioning and history leaving the controller (issues 774, 1126, 1952, 1953, 1954, 1955, 1289) #133
@@ -611,6 +611,10 @@ func healthChecks(t *testing.T) {
|
||||
bucket("mesh-provisioner", "registry")}},
|
||||
{Module: "mesh-vault", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}},
|
||||
{Module: "postgres", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}},
|
||||
// ADR 0571 clauses 2, 4 (issue 1955): the artifact store's seat says artifacts-gone; the assigner keeps seat-holders.
|
||||
{Module: "artifact-store-tools", Holds: []Seat{{Name: "artifact-store", Emits: []string{"artifacts-gone"}}},
|
||||
Reads: []string{delivery.SeatHoldersRead}},
|
||||
{Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}},
|
||||
{Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}},
|
||||
Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}},
|
||||
{Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools",
|
||||
@@ -640,8 +644,11 @@ func healthChecks(t *testing.T) {
|
||||
// Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570
|
||||
// clauses 1, 8).
|
||||
"a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"},
|
||||
"a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"},
|
||||
"a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"},
|
||||
"a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak",
|
||||
"mesh.seat.artifact-store.event.artifacts-gone"},
|
||||
// Only the assigner writes seat-holders.
|
||||
"a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"},
|
||||
"a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"},
|
||||
// Nor mesh-hand-acts' log and warrants, which it alone writes (ADR 0565 clause 10).
|
||||
"a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch",
|
||||
"$KV.mesh-hand-acts_warrants.x", "mesh.log.mesh-hand-acts.acts.x"},
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package catalogue
|
||||
|
||||
// **The artifact store's sweep, out of the controller** (novox/hq ADR 0571 clause 2, issue 1955): the seat
|
||||
// `artifact-store`, held once for the mesh by artifact-store-tools, serves what the controller served of its store
|
||||
// (artifacts, held, mirrors, images, collect) and says `artifacts-gone`. Its keep set is the store's own listing joined
|
||||
// with every want's fingerprint and each module's five newest builds from the builder's log (ADR 0526 §1); it lets go
|
||||
// of nothing until the controller's lease names a holder that is no controller.
|
||||
|
||||
// ArtifactStoreSeat is the seat artifact-store-tools holds.
|
||||
const ArtifactStoreSeat = "artifact-store"
|
||||
|
||||
// artifactStoreVerbs are the seat's verbs, with the controller's arguments so a caller changes only the seat it asks.
|
||||
func artifactStoreVerbs() []Verb {
|
||||
return []Verb{
|
||||
{Name: "artifacts", Description: "Every artifact the mesh made that lives in its artifact store, each with its " +
|
||||
"repository, digest, kind (image or archive) and state: kept, and why (a want names it, or one of the five " +
|
||||
"newest builds of its module in the builder's log), eligible (made, kept for no reason, not yet let go), or " +
|
||||
"unrecorded (in the store and named by nothing the mesh made: never removed). Reads only.",
|
||||
Input: schema(map[string]string{
|
||||
"repository": "only this repository's references, as <module>/<artifact> (the counts stay whole)",
|
||||
"collected": "\"true\": list the collected references too (counted always)",
|
||||
}, nil, "collected")},
|
||||
{Name: "held", Description: "Whether the artifact store holds each reference given, asked with one HEAD each: " +
|
||||
"held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key. " +
|
||||
"Reads only.",
|
||||
Input: listed(schema(map[string]string{
|
||||
"references": "the artifacts' references, artifact-store://<module>/<artifact>/blobs/sha256:<hex>: a list, " +
|
||||
"or one text separated by commas",
|
||||
}, []string{"references"}), "references")},
|
||||
{Name: "collect", Description: "The artifact store's sweep, now: what is eligible is let go of. Without confirm a " +
|
||||
"dry run: what would be let go, nothing held or deleted. With confirm and why: every kept archive held first, " +
|
||||
"then each eligible artifact let go of, oldest first, an index with the platform manifests no kept index " +
|
||||
"names, each deletion recorded in the hand-act log. Bounded by most (500 by default, at most 5000) and 45 " +
|
||||
"seconds; what is left is said. Never touches a digest the mesh did not make.",
|
||||
Input: schema(map[string]string{
|
||||
"confirm": "\"true\": let go of what is eligible (needs why); without it nothing is done",
|
||||
"why": "with confirm: why, required and recorded",
|
||||
"most": "let go of at most this many (500 by default, at most 5000)",
|
||||
}, nil, "confirm")},
|
||||
{Name: "mirrors", Description: "The bases the mesh copied into its artifact store for builds to stand on, each " +
|
||||
"with its state: kept because a kept build stood on it, or eligible. With record and confirm and why, copies " +
|
||||
"no record names are recorded as the mesh's. Deletes nothing.",
|
||||
Input: schema(map[string]string{
|
||||
"record": "references to record, artifact-store://<repository>@sha256:<hex>, separated by spaces or commas",
|
||||
"confirm": "\"true\": record them (needs why); without it nothing is recorded",
|
||||
"why": "with confirm: why, required and recorded",
|
||||
}, nil, "confirm")},
|
||||
{Name: "images", Description: "Every container image one machine's want and have name, each with the resources " +
|
||||
"naming it: what the machine keeps when it prunes its images. Reads only.",
|
||||
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,10 @@ func placementVerbs() []Verb {
|
||||
"history, one node's or one module's changes in order: a revision the state keeps whole, an older one as the " +
|
||||
"proposal's changes from its placement-changed event.",
|
||||
Input: schema(map[string]string{"node": "one node", "module": "one module: the nodes it is placed on",
|
||||
"history": "\"true\": the changes in order, from the stream and the state's revisions"}, nil, "history")},
|
||||
"history": "\"true\": the changes in order, from the stream and the state's revisions",
|
||||
// ADR 0571 clause 4: who holds which seat on which node, from the state seat-holders.
|
||||
"seats": "\"true\": who holds each seat, and on which node (the state seat-holders); with node, that node's"},
|
||||
nil, "history", "seats")},
|
||||
{Name: PlacementVerbProposals, Optional: true, Description: "Every open proposal of placement: its changes, " +
|
||||
"the revision of each key it was laid over, its candidates, every node's answer, the differences computed and " +
|
||||
"answered, its verdict and its ask. Narrowed to one by id.",
|
||||
|
||||
@@ -303,6 +303,9 @@ var defaultSeats = append([]Seat{
|
||||
// policy and the bus step at the assigner, and the hand-act log.
|
||||
{Name: WantsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"policy-changed"}, Serves: wantsVerbs()},
|
||||
{Name: HandActsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"act-recorded"}, Serves: handActsVerbs()},
|
||||
// The artifact store's sweep (novox/hq ADR 0571 clause 2, artifact_store_seat.go).
|
||||
{Name: ArtifactStoreSeat, Scope: ScopeMesh, Decision: "novox/hq ADR 0571", Emits: []string{"artifacts-gone"},
|
||||
Serves: artifactStoreVerbs()},
|
||||
// Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a
|
||||
// delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight.
|
||||
// They keep their names until that migration is done deliberately, apart from the node-* pass.
|
||||
|
||||
@@ -76,9 +76,11 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// And ADR 0565 clause 1's two.
|
||||
{WantsSeat, "upgrade,bus", "policy-changed"},
|
||||
{HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"},
|
||||
// ADR 0571 clause 2: the artifact store's sweep.
|
||||
{ArtifactStoreSeat, "artifacts,held,collect,mirrors,images", "artifacts-gone"},
|
||||
} {
|
||||
s, ok := SeatNamed(c.seat)
|
||||
if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 056") ||
|
||||
if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 05") ||
|
||||
strings.Join(VerbNames(s.Serves), ",") != c.verbs || strings.Join(s.Emits, ",") != c.emits {
|
||||
t.Errorf("the health seat %s is %v, %+v", c.seat, ok, s)
|
||||
}
|
||||
@@ -120,9 +122,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
// Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1);
|
||||
// fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts
|
||||
// (novox/hq ADR 0565 clause 1).
|
||||
if len(Seats()) != 55 {
|
||||
t.Errorf("the mesh defines %d seats rather than 55; the set is closed, so a change here is "+
|
||||
// (novox/hq ADR 0565 clause 1); fifty-six with artifact-store (novox/hq ADR 0571 clause 2).
|
||||
if len(Seats()) != 56 {
|
||||
t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,44 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) {
|
||||
if _, err := EncodeNodeHealth(NodeHealth{}); err == nil {
|
||||
t.Error("a report naming no node was encoded")
|
||||
}
|
||||
// ADR 0571 clause 4: who holds which seat, from placement; a node seat by its node.
|
||||
held := SeatHoldersOf(map[string]Placement{
|
||||
"ace": {Node: "ace", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}},
|
||||
"novox": {Node: "novox", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"}}},
|
||||
})
|
||||
keys := map[string][]byte{}
|
||||
for k, v := range held {
|
||||
keys[k], _ = json.Marshal(v)
|
||||
}
|
||||
if h, err := HoldersOf(keys, "mesh-dns-resolver", ""); err != nil || len(h) != 2 || h[0].Node != "ace" {
|
||||
t.Errorf("a replicated seat's holders are %v, %v", h, err)
|
||||
}
|
||||
if h, _ := HoldersOf(keys, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 3 {
|
||||
t.Errorf("a node seat's holder is %v, keys %v", h, held)
|
||||
}
|
||||
if h, _ := HoldersOf(keys, "node-files", "novox"); h != nil {
|
||||
t.Errorf("a node seat nobody holds there is held by %v", h)
|
||||
}
|
||||
// ADR 0571 clause 1: the builder's log, a start and an end, and the duration between them.
|
||||
at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
||||
start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Node: "ace", At: at}
|
||||
end := start
|
||||
end.Event, end.At, end.Outcome, end.Artifacts = BuildEndedEvent, at.Add(3*time.Minute), "done", []BuiltDigest{{Name: "image", Digest: "sha256:1"}}
|
||||
for _, e := range []BuildLogEntry{start, end} {
|
||||
raw, err := EncodeBuildLogEntry(e)
|
||||
back, derr := DecodeBuildLogEntry(raw)
|
||||
if err != nil || derr != nil || back.Event != e.Event || back.Outcome != e.Outcome {
|
||||
t.Errorf("%s read back as %+v, %v %v", e.Event, back, err, derr)
|
||||
}
|
||||
}
|
||||
if d, ok := BuildDuration(start, end); !ok || d != 3*time.Minute {
|
||||
t.Errorf("the build took %s, %v", d, ok)
|
||||
}
|
||||
bad := end
|
||||
bad.Outcome = ""
|
||||
if _, err := EncodeBuildLogEntry(bad); err == nil {
|
||||
t.Error("an end with no outcome was encoded")
|
||||
}
|
||||
if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil {
|
||||
t.Errorf("a held value's key is %q, %v", k, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **Who holds which seat, and the builder's log** (novox/hq ADR 0571 clauses 1 and 4, issue 1955).
|
||||
|
||||
// SeatHoldersState is the state the seat mesh-placement's holder writes of who holds which seat on which node (clause 4):
|
||||
// one key per seat held once for the mesh, `<seat>`, and one per node of a seat held on every node, `<seat>.<node>`.
|
||||
const (
|
||||
SeatHoldersState = "seat-holders"
|
||||
SeatHoldersBucket = AssignerModule + "_" + SeatHoldersState
|
||||
SeatHoldersRead = AssignerModule + "." + SeatHoldersState
|
||||
)
|
||||
|
||||
// SeatHolder is one module holding a seat on a node.
|
||||
type SeatHolder struct {
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
}
|
||||
|
||||
// SeatHolders is one key of seat-holders: the seat, its scope, the node for a node seat's key, and its holders
|
||||
// (several only for a replicated mesh seat), sorted.
|
||||
type SeatHolders struct {
|
||||
V int `json:"v"`
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Holders []SeatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// SeatHoldersKey is a seat's key: `<seat>` for a seat held once for the mesh (node empty), `<seat>.<node>` for a node
|
||||
// seat on one node.
|
||||
func SeatHoldersKey(seat, node string) (string, error) {
|
||||
if err := checkName("seat", seat); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if node == "" {
|
||||
return seat, nil
|
||||
}
|
||||
if err := checkName("node", node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return seat + "." + node, nil
|
||||
}
|
||||
|
||||
// SeatHoldersOf is the state composed from every node's placement (ADR 0567 §1): each holding by the seat it claims,
|
||||
// keyed as SeatHoldersKey says, with the module holding it and its node. Deterministic, so a writer writes only a key
|
||||
// that differs.
|
||||
func SeatHoldersOf(placements map[string]Placement) map[string]SeatHolders {
|
||||
out := map[string]SeatHolders{}
|
||||
nodes := make([]string, 0, len(placements))
|
||||
for n := range placements {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
for _, h := range placements[n].Holds {
|
||||
node := ""
|
||||
if h.Scope == "node" {
|
||||
node = n
|
||||
}
|
||||
key, err := SeatHoldersKey(h.Claim, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sh := out[key]
|
||||
sh.V, sh.Seat, sh.Scope, sh.Node = Version, h.Claim, h.Scope, node
|
||||
sh.Holders = append(sh.Holders, SeatHolder{Module: h.Module, Node: n})
|
||||
out[key] = sh
|
||||
}
|
||||
}
|
||||
for k, sh := range out {
|
||||
sort.Slice(sh.Holders, func(i, j int) bool {
|
||||
return sh.Holders[i].Node+"/"+sh.Holders[i].Module < sh.Holders[j].Node+"/"+sh.Holders[j].Module
|
||||
})
|
||||
out[k] = sh
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HoldersOf reads who holds seat (on node, for a node seat) from the state's keys as a reader watched them: the node's
|
||||
// key first, then the seat's. None when nobody holds it.
|
||||
func HoldersOf(keys map[string][]byte, seat, node string) ([]SeatHolder, error) {
|
||||
for _, n := range []string{node, ""} {
|
||||
key, err := SeatHoldersKey(seat, n)
|
||||
if err != nil {
|
||||
if n == "" {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
raw, ok := keys[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var sh SeatHolders
|
||||
if err := json.Unmarshal(raw, &sh); err != nil {
|
||||
return nil, fmt.Errorf("delivery: the holders of %s are not readable: %w", key, err)
|
||||
}
|
||||
if sh.V != Version {
|
||||
return nil, fmt.Errorf("delivery: the holders of %s are of version %d, and this reads %d", key, sh.V, Version)
|
||||
}
|
||||
return sh.Holders, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// The builder's log (clause 1): a stream of its own, `log`, one entry when a build of a fingerprint starts and one when
|
||||
// it ends, kept 30 days.
|
||||
const (
|
||||
BuilderLogStream = "log"
|
||||
BuildStartedEvent = "build.started"
|
||||
BuildEndedEvent = "build.ended"
|
||||
// BuilderLogDays is how long the log keeps its entries.
|
||||
BuilderLogDays = 30
|
||||
)
|
||||
|
||||
// BuiltDigest is one artifact a build ended with, by its name and digest.
|
||||
type BuiltDigest struct {
|
||||
Name string `json:"name"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, the build node and
|
||||
// the time; at the end, the outcome and each artifact's digest.
|
||||
type BuildLogEntry struct {
|
||||
V int `json:"v"`
|
||||
Event string `json:"event"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
Source string `json:"source"`
|
||||
Node string `json:"node"`
|
||||
At time.Time `json:"at"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Artifacts []BuiltDigest `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// EncodeBuildLogEntry is e as JSON at Version; it refuses an entry a reader could not place.
|
||||
func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) {
|
||||
e.V = Version
|
||||
switch {
|
||||
case e.Event != BuildStartedEvent && e.Event != BuildEndedEvent:
|
||||
return nil, fmt.Errorf("delivery: a build log entry is %s or %s, not %q", BuildStartedEvent, BuildEndedEvent, e.Event)
|
||||
case e.Module == "" || e.FP == "" || e.Node == "" || e.At.IsZero():
|
||||
return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, node and time")
|
||||
case e.Event == BuildEndedEvent && e.Outcome == "":
|
||||
return nil, fmt.Errorf("delivery: a build's end names its outcome")
|
||||
case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Artifacts) > 0):
|
||||
return nil, fmt.Errorf("delivery: a build's start names no outcome and no artifact")
|
||||
}
|
||||
return json.Marshal(e)
|
||||
}
|
||||
|
||||
// DecodeBuildLogEntry reads one entry.
|
||||
func DecodeBuildLogEntry(data []byte) (BuildLogEntry, error) {
|
||||
var e BuildLogEntry
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return BuildLogEntry{}, fmt.Errorf("delivery: a build log entry is not readable: %w", err)
|
||||
}
|
||||
if e.V != Version {
|
||||
return BuildLogEntry{}, fmt.Errorf("delivery: a build log entry of version %d, and this reads %d", e.V, Version)
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// BuildDuration is a build's duration, its end less its start (clause 1): false unless both are the same module's
|
||||
// build of the same fingerprint, the start first.
|
||||
func BuildDuration(start, end BuildLogEntry) (time.Duration, bool) {
|
||||
if start.Event != BuildStartedEvent || end.Event != BuildEndedEvent || start.Module != end.Module || start.FP != end.FP ||
|
||||
end.At.Before(start.At) {
|
||||
return 0, false
|
||||
}
|
||||
return end.At.Sub(start.At), true
|
||||
}
|
||||
Reference in New Issue
Block a user