diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 36c28b901..5c91bee57 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -38,6 +38,8 @@ type Holder struct { Source SourceStore // Owed is the gate's bucket owed: what a transition owes the forge, written as keys (owed.go). Owed OwedStore + // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. + Records *Records // Pulls keeps the pull requests' keys on the bus (pullkey.go); nil keeps none. Pulls PullKeys // Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 6d86d423a..2d150687a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -73,6 +73,7 @@ func main() { Conditions: kvConditions{}, } h.init() + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention} l := &listening{now: "not yet: starting"} go run(h, l) if err := announceKept(stdio.Keeps); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go new file mode 100644 index 000000000..b838552d2 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -0,0 +1,507 @@ +package main + +// The delivery record (novox/hq ADR 0656 §7, design 47 *The delivery record*): one per milestone's delivery, a ledger +// instance of the gate's own (ADR 0595's template ledger, ADR 0597) that the gate alone writes. It holds the group's state +// (the heads, the verdicts, the checks and the asks, which `merges` held), the trace (the milestone, its issues, its pull +// requests, their commits and the graph revisions the delivery moved between) and what happened to each entry on each +// node. It is keyed by its milestone and its heads, so a switch or a delivery of milestone G again finds a record already +// there and leaves it as it is. Its layout is the gate's own (design 20 *Where a seat's code goes*: "logic and state +// layouts one module owns"); another module reads it through the seat's verbs, never the ledger (ADR 0596). + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// RecordsLedger is the local name the gate asks its ledger for in `requires`. +const RecordsLedger = "records" + +// RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate +// changes how it writes them. +const RecordForm = 1 + +// The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, +// since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 +// KiB). +const ( + TypeRecord = "record" + TypeHeld = "held" +) + +// The operations of a record, each one effect (ADR 0597 §2), and of the month's list. The manifest's `requires` declares +// exactly these (record_test.go). +const ( + OpOpen = "open" // set-fields: the milestone, the heads, when, forge-made, the form + OpGroup = "group" // set-fields: the group's state + OpTrace = "trace" // set-fields: the trace + OpOutcome = "outcome" // set-entry into outcomes, by the entry, the node and the step + OpAway = "away" // set-entry into away, by node + OpBack = "back" // remove from away, by node + OpPin = "pin" // set-entry into pinned, by revision + OpFree = "free" // remove from pinned, by revision + OpEnd = "end" // set-fields: ended + OpRemove = "remove" // remove the instance, at its retention + OpHold = "hold" // a month's list: set-entry into records, by instance + OpDrop = "drop" // a month's list: remove from records, by instance +) + +// RecordOperations are a record's operations, in the order the manifest's type allows them. +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpEnd, OpRemove} + +// HeldOperations are the month's list's operations. +var HeldOperations = []string{OpHold, OpDrop} + +// DefaultRecordRetention is how long a record is kept when the setting record-retention gives none (design 47: 90 days +// covers every trace the operator has asked of a past delivery so far). +const DefaultRecordRetention = 90 * 24 * time.Hour + +// Record is a delivery record's state, as apply derives it from its operations. +type Record struct { + // Instance is the record's path in the ledger; not in the state. + Instance string `json:"-"` + // Sequence is the last operation the state holds; not in the state. + Sequence uint64 `json:"-"` + + Milestone string `json:"milestone"` + // Heads are the heads the record is for: each member's id (owner/repository@commit) to its full commit. + Heads map[string]string `json:"heads"` + Opened time.Time `json:"opened"` + // ForgeMade is a record opened for a merge the forge made with no record's merge-pull (design 47 *What the gate owes + // the forge*): it has no Approve. + ForgeMade bool `json:"forge-made,omitempty"` + Form int `json:"form,omitempty"` + + Group *RecordGroup `json:"group,omitempty"` + Trace *RecordTrace `json:"trace,omitempty"` + // Outcomes are what happened to each entry on each node, by OutcomeKey. + Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` + // Away are the nodes away, by node, each with since when. + Away map[string]NodeAway `json:"away,omitempty"` + // Pinned are the graph revisions the record points at, by revision. + Pinned map[string]PinnedRevision `json:"pinned,omitempty"` + Ended *RecordEnded `json:"ended,omitempty"` +} + +// RecordGroup is the group's state as the record keeps it: what `merges` held for the group, and its members. +type RecordGroup struct { + Group string `json:"group"` + Members []string `json:"members,omitempty"` + // Ask is the key of the Approve's ask; Approved the heads it covers, `@` to the full head. + Ask string `json:"ask,omitempty"` + Approved map[string]string `json:"approved,omitempty"` + // Pulls are each member's merge, as merging, merged or refused, with the commit or the forge's words. + Pulls []RecordPull `json:"pulls,omitempty"` +} + +// RecordPull is one member's merge. +type RecordPull struct { + Repository string `json:"repository"` + Number int `json:"number"` + Head string `json:"head"` + State string `json:"state"` + Commit string `json:"commit,omitempty"` + Words string `json:"words,omitempty"` +} + +// RecordTrace is the trace: the milestone's issues and pull requests, their commits, and the graph revisions. +type RecordTrace struct { + Issues []int `json:"issues,omitempty"` + Pulls []string `json:"pulls,omitempty"` + Commits []string `json:"commits,omitempty"` + // Source is the source revision the merge moved; From and To the assignment graph revisions it moved between. + Source string `json:"source,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` +} + +// The outcomes an entry has on a node (ADR 0656 §7, design 47 *The delivery record*). +const ( + OutcomeTakenIn = "taken-in" + OutcomeBuilt = "built" + OutcomeChecked = "checked" + OutcomeComposed = "composed" + OutcomeWanted = "wanted" + OutcomeRunning = "running-healthy" + OutcomeHeld = "held" + OutcomeFailed = "failed" + OutcomeAway = "away" +) + +// RecordOutcome is what happened to one entry on one node at one step, with the event it was taken from. +type RecordOutcome struct { + Key string `json:"key"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node,omitempty"` + Step string `json:"step"` + Outcome string `json:"outcome"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` + // Event is the id of the event it was taken from (x-event-id), so an event applied twice is known. + Event string `json:"event,omitempty"` +} + +// OutcomeKey is the key of an entry's outcome: its module, its fingerprint, its node and its step, each a token. +func OutcomeKey(module, fingerprint, node, step string) string { + return strings.Join([]string{module, fingerprint, node, step}, "|") +} + +// NodeAway is a node away, and since when. +type NodeAway struct { + Node string `json:"node"` + Since time.Time `json:"since"` +} + +// PinnedRevision is a graph revision the record points at: its half (catalog or assignment) and its revision. +type PinnedRevision struct { + Revision string `json:"revision"` + Half string `json:"half"` +} + +// RecordEnded is how a record ended: when, why, by whom, and the nodes away it ended without. +type RecordEnded struct { + At time.Time `json:"at"` + Why string `json:"why"` + By string `json:"by,omitempty"` + Away []NodeAway `json:"away,omitempty"` +} + +// heldList is a month's list of records: each record opened that month, by instance. +type heldList struct { + Records map[string]heldEntry `json:"records,omitempty"` +} + +type heldEntry struct { + Instance string `json:"instance"` + Opened time.Time `json:"opened"` +} + +// segmentRe is what one part of an instance's path may hold (the tool runner's rule). +var segmentRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// milestoneToken is a milestone as one part of an instance's path: its letters, digits, - and _ kept, every other byte +// written _, and, when that changed it, a short digest of the milestone added, so two milestones never meet. +func milestoneToken(milestone string) string { + var b strings.Builder + for i := 0; i < len(milestone); i++ { + c := milestone[i] + if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' { + b.WriteByte(c) + continue + } + b.WriteByte('_') + } + if b.String() == milestone && milestone != "" { + return milestone + } + sum := sha256.Sum256([]byte(milestone)) + return b.String() + "_" + hex.EncodeToString(sum[:4]) +} + +// HeadsDigest is the digest of a set of heads: each member's id and full commit, sorted, sixteen hex characters. +func HeadsDigest(heads map[string]string) string { + ids := make([]string, 0, len(heads)) + for id := range heads { + ids = append(ids, id) + } + sort.Strings(ids) + h := sha256.New() + for _, id := range ids { + fmt.Fprintf(h, "%s=%s\n", strings.ToLower(id), strings.ToLower(heads[id])) + } + return hex.EncodeToString(h.Sum(nil))[:16] +} + +// RecordInstance is the instance of the record of a milestone at its heads: `record//`. +func RecordInstance(milestone string, heads map[string]string) (string, error) { + if strings.TrimSpace(milestone) == "" { + return "", errors.New("a delivery record names its milestone") + } + if len(heads) == 0 { + return "", fmt.Errorf("the delivery record of %s names no heads", milestone) + } + for id, sha := range heads { + if id == "" || sha == "" { + return "", fmt.Errorf("the delivery record of %s names a head with no id or commit (%q=%q)", milestone, id, sha) + } + } + inst := TypeRecord + "/" + milestoneToken(milestone) + "/" + HeadsDigest(heads) + for _, p := range strings.Split(inst, "/") { + if !segmentRe.MatchString(p) { + return "", fmt.Errorf("the delivery record of %s cannot be named: %q", milestone, inst) + } + } + return inst, nil +} + +// heldInstance is the list of the records opened in a month: `held/2026-11`. +func heldInstance(t time.Time) string { return TypeHeld + "/" + t.UTC().Format("2006-01") } + +// RecordLedger is the gate's ledger: an operation applied to one instance, and one instance's state read. +type RecordLedger interface { + Apply(instance, operation string, body any) (uint64, error) + // Get is the instance's state and the sequence of its last operation; a nil state when it has none. + Get(instance string) (json.RawMessage, uint64, error) +} + +// sdkLedger is the gate's ledger through the runtime: the tool runner's ledger calls, on the gate's own ledger only +// (ADR 0597 §3, §10). Only the tool runner's ledger credential publishes on its history (§9). +type sdkLedger struct{} + +// ledgerCall is the runtime's call on the module's own bus state; a test's stand-in. +var ledgerCall = stdio.BusState + +func (sdkLedger) Apply(instance, operation string, body any) (uint64, error) { + raw, err := json.Marshal(body) + if err != nil { + return 0, err + } + out, err := ledgerCall(buscalls.LedgerApply, buscalls.ApplyRequest{Ledger: RecordsLedger, Instance: instance, + Operation: operation, Body: raw}) + if err != nil { + return 0, err + } + var got buscalls.Applied + if err := json.Unmarshal(out, &got); err != nil { + return 0, fmt.Errorf("the ledger's answer to %s on %s is not readable: %w", operation, instance, err) + } + if !got.Taken { + return 0, fmt.Errorf("the ledger did not take %s on %s", operation, instance) + } + return got.Sequence, nil +} + +func (sdkLedger) Get(instance string) (json.RawMessage, uint64, error) { + out, err := ledgerCall(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: RecordsLedger, Instance: instance}) + if err != nil { + return nil, 0, err + } + var got buscalls.InstanceState + if err := json.Unmarshal(out, &got); err != nil { + return nil, 0, fmt.Errorf("the ledger's state of %s is not readable: %w", instance, err) + } + if len(got.State) == 0 || string(got.State) == "null" { + return nil, got.Sequence, nil + } + return got.State, got.Sequence, nil +} + +// Records keeps the delivery records on the ledger: the one writer of them. +type Records struct { + Ledger RecordLedger + Now func() time.Time + // Retention is how long a record is kept; nil keeps DefaultRecordRetention. + Retention func() time.Duration + // Freed is told each revision a removed record pinned, to say as `freed` (design 47); nil says none. + Freed func(r *Record, p PinnedRevision) +} + +func (rs *Records) now() time.Time { + if rs.Now == nil { + return time.Now().UTC() + } + return rs.Now().UTC() +} + +func (rs *Records) retention() time.Duration { + if rs.Retention == nil { + return DefaultRecordRetention + } + if d := rs.Retention(); d > 0 { + return d + } + return DefaultRecordRetention +} + +// Read is one record, or nil when the ledger holds none at that instance (or one removed). +func (rs *Records) Read(instance string) (*Record, error) { + raw, seq, err := rs.Ledger.Get(instance) + if err != nil || raw == nil { + return nil, err + } + var r Record + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("the delivery record %s is not readable: %w", instance, err) + } + if r.Milestone == "" { + return nil, nil + } + r.Instance, r.Sequence = instance, seq + return &r, nil +} + +// Open opens the record of a milestone at its heads, unless one is there already, which is left as it is (the switch +// and a delivery of milestone G again after a revert make no second record and ask nothing again). Opened says it was +// opened now. Each open removes the records past their retention first (the retention needs no timer: records open as +// deliveries do). +func (rs *Records) Open(milestone string, heads map[string]string, forgeMade bool) (r *Record, opened bool, err error) { + inst, err := RecordInstance(milestone, heads) + if err != nil { + return nil, false, err + } + if r, err = rs.Read(inst); err != nil || r != nil { + return r, false, err + } + now := rs.now() + if err := rs.Sweep(now); err != nil { + return nil, false, fmt.Errorf("the records past their retention cannot be removed, so no record is opened: %w", err) + } + body := map[string]any{"milestone": milestone, "heads": heads, "opened": now, "form": RecordForm} + if forgeMade { + body["forge-made"] = true + } + if _, err := rs.Ledger.Apply(inst, OpOpen, body); err != nil { + return nil, false, err + } + if _, err := rs.Ledger.Apply(heldInstance(now), OpHold, heldEntry{Instance: inst, Opened: now}); err != nil { + return nil, false, fmt.Errorf("the record %s is opened but not listed: %w", inst, err) + } + r, err = rs.Read(inst) + return r, err == nil && r != nil, err +} + +// List is every record kept, opened within the retention and a month more, oldest first. +func (rs *Records) List() ([]*Record, error) { + now := rs.now() + var out []*Record + for _, m := range rs.months(now) { + list, err := rs.heldIn(m) + if err != nil { + return nil, err + } + for _, e := range list { + r, err := rs.Read(e.Instance) + if err != nil { + return nil, err + } + if r != nil { + out = append(out, r) + } + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Opened.Before(out[j].Opened) }) + return out, nil +} + +// months are the months' lists a record within the retention can be in, oldest first, one more at each end. +func (rs *Records) months(now time.Time) []string { + from := now.Add(-rs.retention()).AddDate(0, -1, 0) + var out []string + for m := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC); !m.After(now); m = m.AddDate(0, 1, 0) { + out = append(out, heldInstance(m)) + } + return out +} + +func (rs *Records) heldIn(month string) ([]heldEntry, error) { + raw, _, err := rs.Ledger.Get(month) + if err != nil || raw == nil { + return nil, err + } + var l heldList + if err := json.Unmarshal(raw, &l); err != nil { + return nil, fmt.Errorf("the list of records %s is not readable: %w", month, err) + } + out := make([]heldEntry, 0, len(l.Records)) + for _, e := range l.Records { + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { return out[i].Opened.Before(out[j].Opened) }) + return out, nil +} + +// Sweep removes every record opened longer ago than the retention, and says `freed` for each revision it pinned +// (design 47: "When the retention removes a record, the gate says freed for each revision it pinned"). Nothing else is +// removed to make room. +func (rs *Records) Sweep(now time.Time) error { + cut := now.Add(-rs.retention()) + for _, m := range rs.months(now) { + list, err := rs.heldIn(m) + if err != nil { + return err + } + for _, e := range list { + if !e.Opened.Before(cut) { + continue + } + r, err := rs.Read(e.Instance) + if err != nil { + return err + } + if r != nil { + if _, err := rs.Ledger.Apply(e.Instance, OpRemove, map[string]any{}); err != nil { + return err + } + if rs.Freed != nil { + for _, p := range sortedPins(r) { + rs.Freed(r, p) + } + } + } + if _, err := rs.Ledger.Apply(m, OpDrop, map[string]any{"instance": e.Instance}); err != nil { + return err + } + } + } + return nil +} + +func sortedPins(r *Record) []PinnedRevision { + out := make([]PinnedRevision, 0, len(r.Pinned)) + for _, p := range r.Pinned { + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Revision < out[j].Revision }) + return out +} + +// SetGroup writes the record's group state, unless it holds that state already. +func (rs *Records) SetGroup(r *Record, g RecordGroup) error { + if r.Group != nil && reflect.DeepEqual(*r.Group, g) { + return nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) + if err != nil { + return err + } + r.Group, r.Sequence = &g, seq + return nil +} + +// Outcome writes one entry's outcome on one node, unless the record holds it already: an event applied twice changes +// nothing (design 47). Applied says it was written now. +func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) { + if o.Module == "" || o.Step == "" || o.Outcome == "" { + return false, fmt.Errorf("an outcome names its module, its step and what happened (%+v)", o) + } + o.Key = OutcomeKey(o.Module, o.Fingerprint, o.Node, o.Step) + if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) { + return false, nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpOutcome, o) + if err != nil { + return false, err + } + if r.Outcomes == nil { + r.Outcomes = map[string]RecordOutcome{} + } + r.Outcomes[o.Key], r.Sequence = o, seq + return true, nil +} + +// sameOutcome is whether two outcomes say the same, whenever and from whichever event they were taken. +func sameOutcome(a, b RecordOutcome) bool { + a.At, b.At, a.Event, b.Event = time.Time{}, time.Time{}, "", "" + return a == b +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go new file mode 100644 index 000000000..b101b362f --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -0,0 +1,305 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// ledgerFake plays the tool runner's ledger for the gate's `records`, from the types the manifest declares: each operation +// checked against its type and folded by its effect as apply does (ADR 0597 §2), so a record the manifest's types could not +// hold fails here. +type ledgerFake struct { + t *testing.T + types map[string]catalogue.LedgerType + states map[string]json.RawMessage + seq map[string]uint64 + history []string + failOn string +} + +func newLedgerFake(t *testing.T) *ledgerFake { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Requires []json.RawMessage `json:"requires"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + f := &ledgerFake{t: t, types: map[string]catalogue.LedgerType{}, states: map[string]json.RawMessage{}, seq: map[string]uint64{}} + for _, r := range m.Requires { + var e catalogue.BusStateEntry + if json.Unmarshal(r, &e) == nil && e.Name == RecordsLedger && e.Storage == catalogue.StorageLedger { + for _, ty := range e.Types { + f.types[ty.Name] = ty + } + } + } + if len(f.types) == 0 { + t.Fatalf("the manifest asks for no ledger %s in requires", RecordsLedger) + } + return f +} + +func (f *ledgerFake) Apply(instance, operation string, body any) (uint64, error) { + if f.failOn == operation { + return 0, fmt.Errorf("the ledger is away") + } + ty, ok := f.types[strings.SplitN(instance, "/", 2)[0]] + if !ok { + return 0, fmt.Errorf("no type for %s", instance) + } + var op *catalogue.LedgerOperation + for i := range ty.Operations { + if ty.Operations[i].Name == operation { + op = &ty.Operations[i] + } + } + if op == nil { + return 0, fmt.Errorf("the type %s allows no operation %q", ty.Name, operation) + } + raw, _ := json.Marshal(body) + in := map[string]any{} + if err := json.Unmarshal(raw, &in); err != nil { + return 0, err + } + for _, field := range op.Fields { + if _, known := ty.Schema[field]; !known { + return 0, fmt.Errorf("the operation %s copies %s, which the type %s's schema lacks", operation, field, ty.Name) + } + } + for name := range in { + if _, known := ty.Schema[name]; !known && !slices.Contains(op.Fields, name) { + f.t.Errorf("the gate's %s on %s carries %s, which the manifest's type %s neither holds nor copies", operation, instance, name, ty.Name) + } + } + state := map[string]any{} + if old := f.states[instance]; old != nil { + _ = json.Unmarshal(old, &state) + } + copied := map[string]any{} + for _, field := range op.Fields { + if v, ok := in[field]; ok { + copied[field] = v + } + } + f.seq[instance]++ + switch op.Effect { + case catalogue.EffectSetFields: + for k, v := range copied { + state[k] = v + } + case catalogue.EffectSetEntry: + key, ok := in[op.Key].(string) + if !ok { + return 0, fmt.Errorf("%s names no entry by %s", operation, op.Key) + } + m, _ := state[op.Into].(map[string]any) + if m == nil { + m = map[string]any{} + } + m[key] = copied + state[op.Into] = m + case catalogue.EffectRemove: + if op.Into == "" { + f.states[instance] = json.RawMessage("null") + f.history = append(f.history, instance+" "+operation) + return f.seq[instance], nil + } + if m, ok := state[op.Into].(map[string]any); ok { + delete(m, in[op.Key].(string)) + } + default: + return 0, fmt.Errorf("effect %s", op.Effect) + } + out, _ := json.Marshal(state) + f.states[instance] = out + f.history = append(f.history, instance+" "+operation) + return f.seq[instance], nil +} + +func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) { + s := f.states[instance] + if s == nil || string(s) == "null" { + return nil, f.seq[instance], nil + } + return s, f.seq[instance], nil +} + +// W11.1 (ADR 0656 §7, design 47 *The delivery record*): one record per milestone's delivery, a ledger instance keyed by its +// milestone and its heads, kept for its retention, its operations those the manifest's ledger types allow. +func TestTheDeliveryRecordIsALedgerInstanceKeyedByMilestoneAndHeads(t *testing.T) { + heads := map[string]string{"novox/app@111111111111": strings.Repeat("1", 40), "novox/lib@222222222222": strings.Repeat("2", 40)} + same := map[string]string{"novox/lib@222222222222": strings.Repeat("2", 40), "novox/app@111111111111": strings.Repeat("1", 40)} + other := map[string]string{"novox/app@111111111111": strings.Repeat("3", 40), "novox/lib@222222222222": strings.Repeat("2", 40)} + at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + + t.Run("an instance per milestone and heads, a path the tool runner takes", func(t *testing.T) { + a, err := RecordInstance("47/G", heads) + if err != nil { + t.Fatal(err) + } + b, _ := RecordInstance("47/G", same) + c, _ := RecordInstance("47/G", other) + d, _ := RecordInstance("47_G", heads) + if a != b || a == c || a == d { + t.Fatalf("instances %s %s %s %s: one per milestone and heads, whatever the heads' order", a, b, c, d) + } + for _, p := range strings.Split(a, "/") { + if !segmentRe.MatchString(p) { + t.Fatalf("%s has a part the tool runner refuses", a) + } + } + if _, err := RecordInstance("", heads); err == nil { + t.Fatal("a record with no milestone is named") + } + if _, err := RecordInstance("47/G", nil); err == nil { + t.Fatal("a record with no heads is named") + } + }) + + t.Run("opened once; a second open leaves it as it is", func(t *testing.T) { + l := newLedgerFake(t) + now := at + rs := &Records{Ledger: l, Now: func() time.Time { return now }} + r, opened, err := rs.Open("47/G", heads, false) + if err != nil || !opened || r == nil || r.Milestone != "47/G" || len(r.Heads) != 2 || !r.Opened.Equal(at) || r.Form != RecordForm { + t.Fatalf("open: %+v %v %v", r, opened, err) + } + now = at.Add(time.Hour) + again, opened, err := rs.Open("47/G", same, false) + if err != nil || opened || again.Instance != r.Instance || !again.Opened.Equal(at) { + t.Fatalf("a second open of the same heads: %+v %v %v", again, opened, err) + } + list, err := rs.List() + if err != nil || len(list) != 1 || list[0].Instance != r.Instance { + t.Fatalf("list %+v %v", list, err) + } + }) + + t.Run("an outcome applied twice changes nothing", func(t *testing.T) { + l := newLedgerFake(t) + rs := &Records{Ledger: l, Now: func() time.Time { return at }} + r, _, _ := rs.Open("47/G", heads, false) + o := RecordOutcome{Module: "mesh-delivery-gate", Fingerprint: "src2:abcd", Node: "ace", Step: "build", Outcome: OutcomeBuilt, At: at, Event: "e1"} + if applied, err := rs.Outcome(r, o); err != nil || !applied { + t.Fatalf("first: %v %v", applied, err) + } + n := len(l.history) + o.At = at.Add(time.Minute) + if applied, err := rs.Outcome(r, o); err != nil || applied || len(l.history) != n { + t.Fatalf("the same event again: %v %v, %d operations (was %d)", applied, err, len(l.history), n) + } + o.Event = "e2" + if applied, _ := rs.Outcome(r, o); applied { + t.Fatal("the same outcome from another event is written again") + } + o.Outcome, o.Why, o.Event = OutcomeFailed, "the build failed", "e3" + if applied, _ := rs.Outcome(r, o); !applied { + t.Fatal("a changed outcome is not written") + } + back, _ := rs.Read(r.Instance) + if got := back.Outcomes[OutcomeKey("mesh-delivery-gate", "src2:abcd", "ace", "build")]; got.Outcome != OutcomeFailed || got.Why == "" { + t.Fatalf("the ledger holds %+v", back.Outcomes) + } + g := RecordGroup{Group: "feat/x", Members: []string{"novox/app@111111111111"}, Ask: "merge.feat/x.h"} + if err := rs.SetGroup(back, g); err != nil { + t.Fatal(err) + } + n = len(l.history) + if err := rs.SetGroup(back, g); err != nil || len(l.history) != n { + t.Fatalf("the same group state is written again: %v", err) + } + }) + + t.Run("a record past its retention is removed at the next open and frees what it pinned", func(t *testing.T) { + l := newLedgerFake(t) + now := at + var freed []string + rs := &Records{Ledger: l, Now: func() time.Time { return now }, Retention: func() time.Duration { return 90 * 24 * time.Hour }, + Freed: func(r *Record, p PinnedRevision) { freed = append(freed, r.Milestone+" "+p.Revision) }} + old, _, _ := rs.Open("46/F", heads, false) + if _, err := l.Apply(old.Instance, OpPin, PinnedRevision{Revision: "cat-1", Half: "catalog"}); err != nil { + t.Fatal(err) + } + now = at.Add(89 * 24 * time.Hour) + kept, _, _ := rs.Open("47/G", heads, false) + if r, _ := rs.Read(old.Instance); r == nil || len(freed) != 0 { + t.Fatalf("a record within its retention is removed (%v)", freed) + } + now = at.Add(91 * 24 * time.Hour) + if _, _, err := rs.Open("47/H", heads, false); err != nil { + t.Fatal(err) + } + if r, _ := rs.Read(old.Instance); r != nil { + t.Fatal("a record past its retention is kept") + } + if !slices.Equal(freed, []string{"46/F cat-1"}) { + t.Fatalf("freed %v", freed) + } + list, _ := rs.List() + if len(list) != 2 || list[0].Instance != kept.Instance { + t.Fatalf("list %+v", list) + } + }) + + t.Run("the setting record-retention, in days", func(t *testing.T) { + dir := t.TempDir() + for _, c := range []struct { + file string + want time.Duration + bad bool + }{ + {`{}`, DefaultRecordRetention, false}, + {`{"record-retention": 30}`, 30 * 24 * time.Hour, false}, + {`{"record-retention": 0}`, 0, true}, + {`{"record-retention": 1.5}`, 0, true}, + {`{"record-retention": "90"}`, 0, true}, + } { + p := filepath.Join(dir, "settings.json") + if err := os.WriteFile(p, []byte(c.file), 0o600); err != nil { + t.Fatal(err) + } + s, err := readSettings(p) + if (err != nil) != c.bad { + t.Fatalf("%s: %v", c.file, err) + } + if c.bad { + continue + } + h := &Holder{Settings: func() Settings { return s }} + if got := h.recordRetention(); got != c.want { + t.Fatalf("%s: %s, wanted %s", c.file, got, c.want) + } + } + }) +} + +// W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. +func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { + l := newLedgerFake(t) + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations} { + var got []string + for _, op := range l.types[name].Operations { + got = append(got, op.Name) + } + if !slices.Equal(got, want) { + t.Errorf("the type %s allows %v, the gate applies %v", name, got, want) + } + } + raw, _ := os.ReadFile(filepath.Join("..", "..", "module.json")) + if !strings.Contains(string(raw), `"max-mib": 768`) { + t.Error("the records ledger is not capped at 768 MiB") + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go index 9d0d016e5..c3217ae2e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go @@ -29,6 +29,9 @@ type Settings struct { // "run the check locally only, but not as a gate, it loses us too much time"): a reviewed pull request is ready and its // Approve asked without a verdict, a trunk's move puts nothing out of date, and a verdict is shown, never waited on. CheckGatesMerge *bool `json:"check-gates-merge"` + // RecordRetentionDays is how many days a delivery record is kept from when it opened (design 47 *The delivery + // record*); nil keeps DefaultRecordRetention. + RecordRetentionDays *int `json:"record-retention"` } // readSettings reads the settings file; a value that is not a whole number of minutes, one or more, is refused. @@ -59,6 +62,17 @@ func readSettings(path string) (Settings, error) { return Settings{}, fmt.Errorf("%s is not a number", name) } } + switch v := loose["record-retention"].(type) { + case nil: + case float64: + n := int(v) + if v < 1 || v > 3650 || v != float64(n) { + return Settings{}, errors.New("record-retention is not a whole number of days, from 1 to 3650") + } + s.RecordRetentionDays = &n + default: + return Settings{}, errors.New("record-retention is not a number") + } switch v := loose["check-gates-merge"].(type) { case nil: case bool: @@ -99,6 +113,17 @@ func (h *Holder) refusalFloor() time.Duration { return minutesOr(h.Settings().RefusalFloorMinutes, DefaultRefusalFloor) } +// recordRetention is how long a delivery record is kept from when it opened. +func (h *Holder) recordRetention() time.Duration { + if h.Settings == nil { + return DefaultRecordRetention + } + if n := h.Settings().RecordRetentionDays; n != nil { + return time.Duration(*n) * 24 * time.Hour + } + return DefaultRecordRetention +} + // authorBound is how long a delivery that needs its author's new head waits before the operator hears of it (§5). func (h *Holder) authorBound() time.Duration { if h.Settings == nil { diff --git a/modules/mesh-delivery-gate/files/settings.json b/modules/mesh-delivery-gate/files/settings.json index 66c66f877..e164ea53c 100644 --- a/modules/mesh-delivery-gate/files/settings.json +++ b/modules/mesh-delivery-gate/files/settings.json @@ -1,5 +1,6 @@ { "refusal-stall-floor-minutes": 10, "author-new-head-minutes": 120, - "check-gates-merge": true + "check-gates-merge": true, + "record-retention": 90 } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index ec98a4044..0c48589f9 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -38,6 +38,204 @@ "uses": [ "operator-channel" ], + "requires": [ + { + "name": "records", + "storage": "ledger", + "template": "ledger", + "options": { + "max-mib": 768 + }, + "types": [ + { + "name": "record", + "schema": { + "milestone": { + "type": "string" + }, + "heads": { + "type": "object" + }, + "opened": { + "type": "string" + }, + "forge-made": { + "type": "boolean" + }, + "form": { + "type": "integer" + }, + "group": { + "type": "object" + }, + "trace": { + "type": "object" + }, + "key": { + "type": "string" + }, + "module": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "node": { + "type": "string" + }, + "step": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "why": { + "type": "string" + }, + "at": { + "type": "string" + }, + "event": { + "type": "string" + }, + "since": { + "type": "string" + }, + "revision": { + "type": "string" + }, + "half": { + "type": "string" + }, + "ended": { + "type": "object" + } + }, + "operations": [ + { + "name": "open", + "effect": "set-fields", + "fields": [ + "milestone", + "heads", + "opened", + "forge-made", + "form" + ], + "sequence": true + }, + { + "name": "group", + "effect": "set-fields", + "fields": [ + "group" + ], + "sequence": true + }, + { + "name": "trace", + "effect": "set-fields", + "fields": [ + "trace" + ], + "sequence": true + }, + { + "name": "outcome", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node", + "step", + "outcome", + "why", + "at", + "event" + ], + "into": "outcomes", + "key": "key", + "sequence": true + }, + { + "name": "away", + "effect": "set-entry", + "fields": [ + "node", + "since" + ], + "into": "away", + "key": "node" + }, + { + "name": "back", + "effect": "remove", + "into": "away", + "key": "node" + }, + { + "name": "pin", + "effect": "set-entry", + "fields": [ + "revision", + "half" + ], + "into": "pinned", + "key": "revision" + }, + { + "name": "free", + "effect": "remove", + "into": "pinned", + "key": "revision" + }, + { + "name": "end", + "effect": "set-fields", + "fields": [ + "ended" + ], + "sequence": true + }, + { + "name": "remove", + "effect": "remove" + } + ] + }, + { + "name": "held", + "schema": { + "instance": { + "type": "string" + }, + "opened": { + "type": "string" + } + }, + "operations": [ + { + "name": "hold", + "effect": "set-entry", + "fields": [ + "instance", + "opened" + ], + "into": "records", + "key": "instance" + }, + { + "name": "drop", + "effect": "remove", + "into": "records", + "key": "instance" + } + ] + } + ] + } + ], "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg": "0.1" }, @@ -105,6 +303,14 @@ "description": "Whether mesh/check gates the merge. True (the default): the operator's Approve is asked, and a merge handed, only on a passing verdict judged against the trunk's head, and a trunk's move puts a verdict out of date (ADR 0447, ADR 0505 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"; developers run the unit and integration levels locally before a pull request): a ready pull request (no review line is waited for, ADR 0659) has its Approve is asked once the check at its heads (a group: of all its heads together) has a verdict, a pass or a failure said in the ask, or after 60 minutes with none, said so (the operator, 2026-10-27 23:54: \"if you don't want me to approve it, don't send the approval request\"); an approved merge waits for no verdict; no merge is refused for a check judged on an older trunk; a later verdict is shown on the pull request and the delivery, and asks a pending Approve again in its words; a group still merges all its members in one pass. Set it false together with the gitea module's check-gates-merge, which lets its merge tool merge an approved head without a passing mesh/check.", "trusted": false, "default": true + }, + "record-retention": { + "type": "number", + "description": "days a delivery record is kept from when it opened (design 47 *The delivery record*): ninety covers every trace asked of a past delivery so far; when a record is removed at its retention, the gate says freed for each graph revision it pinned. Nothing else is removed to make room.", + "minimum": 1, + "maximum": 3650, + "trusted": false, + "default": 90 } }, "resources": [ @@ -123,7 +329,8 @@ "takes-config": [ "refusal-stall-floor-minutes", "author-new-head-minutes", - "check-gates-merge" + "check-gates-merge", + "record-retention" ], "merge": "json", "template": "files/settings.json"