From 6f744d6602daf846caa8dd32cf4b8930f25d8feb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:19:16 +0100 Subject: [PATCH] GT1: the gate's ledger keeps one entry per module and revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record's head entry record/ keeps the record-wide fields and the count of its movers; each mover, the catalog half's too, has its entry mover/ with its pipeline, version and step outcomes per node, found through the index movers on its record's key. An entry over 64 KiB is refused with record-too-large, cleared when written within it or at the record's end; the ledger is capped at 1 GiB. Builds: design 47 H2 §809, §820; H-A-385, H-A-391 --- .../cmd/mesh-delivery-gate/findings.go | 33 ++- .../cmd/mesh-delivery-gate/main.go | 2 +- .../cmd/mesh-delivery-gate/movers.go | 221 ++++++++++++++++++ .../cmd/mesh-delivery-gate/record.go | 101 +++++++- .../cmd/mesh-delivery-gate/record_test.go | 97 +++++++- .../cmd/mesh-delivery-gate/recordevents.go | 1 + .../cmd/mesh-delivery-gate/recordpipeline.go | 6 + .../cmd/mesh-delivery-gate/recordtools.go | 13 +- modules/mesh-delivery-gate/module.json | 127 +++++++--- 9 files changed, 552 insertions(+), 49 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go index 194f71865..1ac846fc4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/findings.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" @@ -28,5 +29,35 @@ type FindingWriter interface { // generated contract of the seat that defines it (A-17). It is started once the runtime serves (A-40). func NewFindingWriter() (*views.HeldWriter[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent], error) { return views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken}}) + Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken, RecordTooLarge}}) +} + +// recordTooLarge raises record-too-large for a delivery record's entry the gate refused over EntryLimit, or clears it when +// the writer holds it (design 47 H2 §809): the record's records hook. +func (h *Holder) recordTooLarge(instance string, bytes int, raised bool) { + if h.Findings == nil { + return + } + held := false + for _, f := range h.Findings.Held() { + if f.Scope == FindingScope && f.ID == instance && f.Kind == RecordTooLarge { + held = true + } + } + var err error + switch { + case raised && !held: + now := h.Now().UTC() + err = h.Findings.Write(context.Background(), meshconditions.Finding{Scope: FindingScope, ID: instance, Kind: RecordTooLarge, + Observer: Module, Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Since: now, At: now, + Summary: fmt.Sprintf("%s would hold %d bytes, over the %d one entry holds: the write is refused", instance, bytes, EntryLimit), + Headline: "A delivery record's entry is too large", Subject: meshconditions.Subject{Scope: FindingScope, ID: instance}, + Explanation: "The gate keeps one entry per record and one per mover, each within 64 KiB (design 47 H2 §809); " + + "this entry would pass it, so the write is refused. It clears when the entry is next written within 64 KiB or the record ends."}) + case !raised && held: + err = h.Findings.Clear(context.Background(), FindingScope, instance, RecordTooLarge, "the entry is written within 64 KiB, or its record ended") + } + if err != nil && h.Logf != nil { + h.Logf("[mesh-delivery] record-too-large of %s is not said: %v", instance, err) + } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 2391c55ee..509b4e256 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -60,7 +60,7 @@ func main() { } h.init() h.SaySeat = stdio.SeatPublish - h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf} + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf, TooLarge: h.recordTooLarge} h.Records.Changed = h.recordChanged h.Forge = seatGit{ask: stdio.Ask} h.Catalog = seatCatalog{ask: stdio.Ask} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go new file mode 100644 index 000000000..8b3fdbf4c --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -0,0 +1,221 @@ +package main + +// A record's entries, one per module (design 47 H2 §809; H-A-385, H-A-391): the head entry `record/` keeps +// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module; each mover, the +// catalog half's too, has its own entry `mover/` with its pipeline, version and the module's step +// outcomes per node (its machines and their running times among them). No entry names two modules. A record is joined +// from its head and the movers the ledger's index `movers` names under the record's key, one query. The gate refuses an +// entry over 64 KiB with the finding record-too-large, cleared when that entry is next written within 64 KiB or at the +// record's end. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// TypeMover is the ledger type of a mover's entry. +const TypeMover = "mover" + +// MoversIndex is the ledger's index of the movers' entries on their record's key (manifest `requires` records' indexes). +const MoversIndex = "movers" + +// The operations of a mover's entry; the manifest's type mover declares exactly these (record_test.go). +const ( + OpMover = "mover" // set-fields: its record's key, its module, its pipeline and its version + // OpEntry and OpOutcome write its entries and outcomes as they did on the whole record. +) + +// MoverOperations are a mover's operations, in the order the manifest's type allows them. +var MoverOperations = []string{OpMover, OpEntry, OpOutcome} + +// OpMovers is the head entry's count of its movers (set-fields movers). +const OpMovers = "movers" + +// EntryLimit is the most one entry of the ledger holds (§809). +const EntryLimit = 64 << 10 + +// RecordTooLarge is the finding of an entry refused over EntryLimit (scope delivery). +const RecordTooLarge = "record-too-large" + +// Mover is one mover's entry, as apply derives it from its operations. +type Mover struct { + // Instance and Sequence are its path in the ledger and its last operation; not in the state. + Instance string `json:"-"` + Sequence uint64 `json:"-"` + + // Record is its record's key, `.`, the index's field. + Record string `json:"record"` + Module string `json:"module"` + Pipeline string `json:"pipeline,omitempty"` + Version string `json:"version,omitempty"` + Entries map[string]RecordEntry `json:"entries,omitempty"` + Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` +} + +// Running are the machines the mover runs healthy on, each with when it was first said. +func (m *Mover) Running() map[string]time.Time { + out := map[string]time.Time{} + for _, o := range m.Outcomes { + if o.Outcome == OutcomeRunning && o.Node != "" { + if at, ok := out[o.Node]; !ok || o.At.Before(at) { + out[o.Node] = o.At + } + } + } + return out +} + +// MoverInstance is the entry of a record's mover: `mover/`, each escaped as a name. +func MoverInstance(milestone, module string) (string, error) { + if strings.TrimSpace(module) == "" { + return "", fmt.Errorf("a mover of the record %s names its module", milestone) + } + inst := TypeMover + "/" + milestoneToken(milestone+"."+module) + if !segmentRe.MatchString(strings.TrimPrefix(inst, TypeMover+"/")) { + return "", fmt.Errorf("the mover %s of %s cannot be named: %q", module, milestone, inst) + } + return inst, nil +} + +// moversKey is the index key that names every mover of a record: its key's token, then the rest. +func moversKey(milestone string) string { + return strings.TrimSuffix(buscalls.IndexKey([]string{milestone}, "x"), ".x") + ".>" +} + +// movers reads the movers' entries of a record through the index, one query, and each entry. +func (rs *Records) movers(milestone string) ([]*Mover, error) { + insts, err := rs.Ledger.Query(MoversIndex, moversKey(milestone)) + if err != nil { + return nil, fmt.Errorf("the movers of the record %s cannot be read: %w", milestone, err) + } + var out []*Mover + for _, inst := range insts { + raw, seq, err := rs.Ledger.Get(inst) + if err != nil { + return nil, err + } + if raw == nil { + continue + } + var m Mover + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("the mover %s is not readable: %w", inst, err) + } + if m.Record != milestone || m.Module == "" { + continue + } + m.Instance, m.Sequence = inst, seq + out = append(out, &m) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out, nil +} + +// join lays a record's movers into it: their entries and outcomes, by module. +func (r *Record) join(ms []*Mover) { + r.movers = map[string]*Mover{} + for _, m := range ms { + r.movers[m.Module] = m + for k, e := range m.Entries { + if r.Entries == nil { + r.Entries = map[string]RecordEntry{} + } + r.Entries[k] = e + } + for k, o := range m.Outcomes { + if r.Outcomes == nil { + r.Outcomes = map[string]RecordOutcome{} + } + r.Outcomes[k] = o + } + } +} + +// Movers are the record's movers' entries, by module. +func (r *Record) Movers() map[string]*Mover { return r.movers } + +// mover is the record's entry of module, opened in the ledger when it has none, and counted on the head entry. +func (rs *Records) mover(r *Record, module string) (*Mover, error) { + if m := r.movers[module]; m != nil { + return m, nil + } + inst, err := MoverInstance(r.Milestone, module) + if err != nil { + return nil, err + } + m := &Mover{Instance: inst, Record: r.Milestone, Module: module} + seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module}) + if err != nil { + return nil, err + } + m.Sequence = seq + if r.movers == nil { + r.movers = map[string]*Mover{} + } + r.movers[module] = m + hseq, err := rs.Ledger.Apply(r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}) + if err != nil { + return m, fmt.Errorf("the mover %s is written and the record %s's count of its movers is not: %w", module, r.Instance, err) + } + r.MoverCount, r.Sequence = len(r.movers), hseq + return m, nil +} + +// SetMover writes a mover's pipeline and version, unless its entry holds them already. +func (rs *Records) SetMover(r *Record, module, pipeline, version string) error { + m, err := rs.mover(r, module) + if err != nil { + return err + } + next := *m + next.Pipeline, next.Version = firstOf(pipeline, m.Pipeline), firstOf(version, m.Version) + if next.Pipeline == m.Pipeline && next.Version == m.Version { + return nil + } + if err := rs.fits(m.Instance, next); err != nil { + return err + } + seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, + "pipeline": next.Pipeline, "version": next.Version}) + if err != nil { + return err + } + m.Pipeline, m.Version, m.Sequence = next.Pipeline, next.Version, seq + rs.fitted(m.Instance) + return nil +} + +// fits refuses an entry whose state would pass EntryLimit, raising record-too-large for it (§809). +func (rs *Records) fits(instance string, state any) error { + raw, err := json.Marshal(state) + if err != nil { + return err + } + if len(raw) <= EntryLimit { + return nil + } + if rs.TooLarge != nil { + rs.TooLarge(instance, len(raw), true) + } + return fmt.Errorf("the delivery record's entry %s would hold %d bytes, over the %d an entry holds: refused (%s)", instance, len(raw), EntryLimit, RecordTooLarge) +} + +// fitted clears record-too-large for an entry written within EntryLimit (TooLarge clears only one it holds). +func (rs *Records) fitted(instance string) { + if rs.TooLarge != nil { + rs.TooLarge(instance, 0, false) + } +} + +// clearTooLarge clears record-too-large for every entry of a record at its end. +func (rs *Records) clearTooLarge(r *Record) { + rs.fitted(r.Instance) + for _, m := range r.movers { + rs.fitted(m.Instance) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 9a1c5da6a..6c095e89c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -31,7 +31,7 @@ const RecordsLedger = "records" // RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate // changes how it writes them. -const RecordForm = 1 +const RecordForm = 2 // The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, // since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 @@ -47,8 +47,8 @@ const ( OpOpen = "open" // set-fields: the milestone, its branch, source revision and merge call, the heads, when, the form OpGroup = "group" // set-fields: the group's state OpTrace = "trace" // set-fields: the trace - OpEntry = "entry" // set-entry into entries: one entry of the graph difference, by its module, fingerprint and node - OpOutcome = "outcome" // set-entry into outcomes, by the entry, the node and the step + OpEntry = "entry" // a mover's: set-entry into entries, one entry of the graph difference, by its module, fingerprint and node + OpOutcome = "outcome" // a mover's: set-entry into outcomes, by the entry, the node and the step OpAway = "away" // set-entry into away, by node OpBack = "back" // remove from away, by node OpEnd = "end" // set-fields: ended @@ -58,7 +58,7 @@ const ( ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpAct, OpEnd} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpAway, OpBack, OpAct, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -99,6 +99,11 @@ type Record struct { Ended *RecordEnded `json:"ended,omitempty"` // Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23). Recovery *RecordRecovery `json:"recovery,omitempty"` + // MoverCount is the count of its movers, each with an entry of its own (design 47 H2 §809); Entries and Outcomes above + // are joined from those entries when read. + MoverCount int `json:"movers,omitempty"` + + movers map[string]*Mover } // RecordRecovery is the field recovery of a record: who named the merge a recovery (the caller the runtime proved), why, @@ -267,6 +272,8 @@ type RecordLedger interface { Get(instance string) (json.RawMessage, uint64, error) // Remove removes a record with its operations and its snapshot through the tool runner's ledger removal. Remove(instance string) error + // Query is the instances a declared index names under key (`>` for the rest). + Query(index, key string) ([]string, error) } // sdkLedger is the gate's ledger through the runtime: the tool runner's ledger calls, on the gate's own ledger only @@ -316,6 +323,22 @@ func (sdkLedger) Remove(instance string) error { return nil } +func (sdkLedger) Query(index, key string) ([]string, error) { + out, err := ledgerCall(buscalls.LedgerQuery, buscalls.QueryRequest{Ledger: RecordsLedger, Index: index, Key: key}) + if err != nil { + return nil, err + } + var got buscalls.QueryAnswer + if err := json.Unmarshal(out, &got); err != nil { + return nil, fmt.Errorf("the ledger's answer to the query of %s is not readable: %w", index, err) + } + insts := make([]string, 0, len(got.Entries)) + for _, e := range got.Entries { + insts = append(insts, e.Instance) + } + return insts, nil +} + func (sdkLedger) Get(instance string) (json.RawMessage, uint64, error) { out, err := ledgerCall(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: RecordsLedger, Instance: instance}) if err != nil { @@ -345,6 +368,8 @@ type Records struct { // Changed is told of every change of a record that owes the forge an act (A-39): a step's outcome applied, the record // ended (owedacts.go); nil owes none. Changed func(r *Record, c RecordChange) + // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. + TooLarge func(instance string, bytes int, raised bool) // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -454,6 +479,11 @@ func (rs *Records) Read(instance string) (*Record, error) { return nil, nil } r.Instance, r.Sequence = instance, seq + ms, err := rs.movers(r.Milestone) + if err != nil { + return nil, err + } + r.join(ms) return &r, nil } @@ -628,6 +658,11 @@ func (rs *Records) Sweep(now time.Time) error { return err } if r != nil { + for _, m := range r.movers { + if err := rs.Ledger.Remove(m.Instance); err != nil { + return err + } + } if err := rs.Ledger.Remove(e.Instance); err != nil { return err } @@ -646,11 +681,17 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { if r.Group != nil && reflect.DeepEqual(*r.Group, g) { return nil } + next := *r + next.Group = &g + if err := rs.fits(r.Instance, next.head()); err != nil { + return err + } seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) if err != nil { return err } r.Group, r.Sequence = &g, seq + rs.fitted(r.Instance) return nil } @@ -659,11 +700,17 @@ func (rs *Records) SetTrace(r *Record, t RecordTrace) error { if r.Trace != nil && reflect.DeepEqual(*r.Trace, t) { return nil } + next := *r + next.Trace = &t + if err := rs.fits(r.Instance, next.head()); err != nil { + return err + } seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t}) if err != nil { return err } r.Trace, r.Sequence = &t, seq + rs.fitted(r.Instance) return nil } @@ -677,14 +724,25 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) { return false, nil } - seq, err := rs.Ledger.Apply(r.Instance, OpOutcome, o) + m, err := rs.mover(r, o.Module) if err != nil { return false, err } + next := *m + next.Outcomes = withEntry(m.Outcomes, o.Key, o) + if err := rs.fits(m.Instance, next); err != nil { + return false, err + } + seq, err := rs.Ledger.Apply(m.Instance, OpOutcome, o) + if err != nil { + return false, err + } + m.Outcomes, m.Sequence = next.Outcomes, seq + rs.fitted(m.Instance) if r.Outcomes == nil { r.Outcomes = map[string]RecordOutcome{} } - r.Outcomes[o.Key], r.Sequence = o, seq + r.Outcomes[o.Key] = o if rs.Changed != nil { rs.Changed(r, RecordChange{What: outcomeWords(o), Why: firstOf(o.Why, "the "+o.Step+" step's event")}) } @@ -697,17 +755,44 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error { if had, ok := r.Entries[e.Key]; ok && had == e { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpEntry, e) + m, err := rs.mover(r, e.Module) if err != nil { return err } + next := *m + next.Entries = withEntry(m.Entries, e.Key, e) + if err := rs.fits(m.Instance, next); err != nil { + return err + } + seq, err := rs.Ledger.Apply(m.Instance, OpEntry, e) + if err != nil { + return err + } + m.Entries, m.Sequence = next.Entries, seq + rs.fitted(m.Instance) if r.Entries == nil { r.Entries = map[string]RecordEntry{} } - r.Entries[e.Key], r.Sequence = e, seq + r.Entries[e.Key] = e return nil } +// withEntry is a copy of m with v under k. +func withEntry[V any](m map[string]V, k string, v V) map[string]V { + out := make(map[string]V, len(m)+1) + for key, val := range m { + out[key] = val + } + out[k] = v + return out +} + +// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's. +func (r Record) head() Record { + r.Entries, r.Outcomes, r.movers = nil, nil, nil + return r +} + // sameOutcome is whether two outcomes say the same, whenever and from whichever event they were taken. func sameOutcome(a, b RecordOutcome) bool { a.At, b.At, a.Event, b.Event = time.Time{}, time.Time{}, "", "" diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index aeb067199..da028b6e5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "sort" "strings" "testing" "time" @@ -150,6 +151,33 @@ func (f *ledgerFake) Remove(instance string) error { return nil } +// Query answers a declared index as the tool runner's watcher keeps it: the key of each instance whose type declares the +// index, from the values of its fields and its path (buscalls.IndexKey), matched with `>` for the rest. +func (f *ledgerFake) Query(index, key string) ([]string, error) { + var out []string + for inst, raw := range f.states { + ty := f.types[strings.SplitN(inst, "/", 2)[0]] + for _, ix := range ty.Indexes { + if ix.Name != index || string(raw) == "null" { + continue + } + state := map[string]any{} + _ = json.Unmarshal(raw, &state) + var vals []string + for _, on := range ix.On { + v, _ := state[on].(string) + vals = append(vals, v) + } + k := buscalls.IndexKey(vals, inst) + if k == key || strings.HasSuffix(key, ">") && strings.HasPrefix(k, strings.TrimSuffix(key, ">")) { + out = append(out, inst) + } + } + } + sort.Strings(out) + return out, nil +} + func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) { s := f.states[instance] if s == nil || string(s) == "null" { @@ -165,6 +193,64 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T same := map[string]string{"novox/lib@4": strings.Repeat("2", 40), "novox/app@3": strings.Repeat("1", 40)} at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + // One entry per module and revision (design 47 H2 §809; H-A-385, H-A-391): 126 movers write 127 entries, none over + // 64 KiB, the head counting 126; an entry made too large is refused with record-too-large, cleared at the record's end. + t.Run("a record of 126 movers writes 127 entries, each within 64 KiB", func(t *testing.T) { + l := newLedgerFake(t) + found := map[string]bool{} + rs := &Records{Ledger: l, Now: func() time.Time { return at }, TooLarge: func(inst string, _ int, raised bool) { + if raised { + found[inst] = true + } else { + delete(found, inst) + } + }} + r, _, err := rs.Open(opening("main", 90, heads)) + if err != nil { + t.Fatal(err) + } + for i := range 126 { + module := fmt.Sprintf("mod-%03d", i) + if err := rs.AddEntry(r, RecordEntry{Module: module, Fingerprint: "src2:f" + module}); err != nil { + t.Fatal(err) + } + if err := rs.SetMover(r, module, "main.90", "0.1.7"); err != nil { + t.Fatal(err) + } + for _, node := range []string{"ace", "g14", "novox", "shanks"} { + if _, err := rs.Outcome(r, RecordOutcome{Module: module, Fingerprint: "src2:f" + module, Node: node, Step: "running", + Outcome: OutcomeRunning, At: at}); err != nil { + t.Fatal(err) + } + } + } + entries := 0 + for inst, raw := range l.states { + if strings.HasPrefix(inst, TypeRecord+"/") || strings.HasPrefix(inst, TypeMover+"/") { + entries++ + if len(raw) > EntryLimit { + t.Errorf("%s holds %d bytes", inst, len(raw)) + } + if strings.HasPrefix(inst, TypeMover+"/") && strings.Count(string(raw), `"module":"mod-`) != 1+1+4 { + t.Errorf("%s names another module: %s", inst, raw) + } + } + } + read, err := rs.Read(r.Instance) + if err != nil || entries != 127 || read.MoverCount != 126 || len(read.Movers()) != 126 || len(read.Outcomes) != 504 || + len(read.Movers()["mod-007"].Running()) != 4 || read.Movers()["mod-007"].Version != "0.1.7" { + t.Fatalf("%d entries, the head counts %d, %d joined, %d outcomes, %v", entries, read.MoverCount, len(read.Movers()), len(read.Outcomes), err) + } + why := strings.Repeat("w", EntryLimit) + if _, err := rs.Outcome(read, RecordOutcome{Module: "mod-001", Fingerprint: "src2:fmod-001", Node: "ace", Step: "build", + Outcome: OutcomeFailed, Why: why, At: at}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 1 { + t.Fatalf("an entry over 64 KiB was taken (%v), findings %v", err, found) + } + if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 { + t.Fatalf("the record's end left %v (%v)", found, err) + } + }) + t.Run("an instance per source revision, a path the tool runner takes", func(t *testing.T) { a, err := RecordInstance(recordMilestone("feat/x", 7)) if err != nil { @@ -322,7 +408,7 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T // W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { l := newLedgerFake(t) - for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations} { + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations} { var got []string for _, op := range l.types[name].Operations { got = append(got, op.Name) @@ -332,8 +418,11 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { } } raw, _ := os.ReadFile(filepath.Join("..", "..", "module.json")) - if !strings.Contains(string(raw), `"max-mib": 768`) { - t.Error("the records ledger is not capped at 768 MiB") + if !strings.Contains(string(raw), `"max-mib": 1024`) { + t.Error("the records ledger is not capped at 1 GiB (design 47 H2 §809)") + } + if ix := l.types[TypeMover].Indexes; len(ix) != 1 || ix[0].Name != MoversIndex || !slices.Equal(ix[0].On, []string{"record"}) { + t.Errorf("the type mover's indexes are %+v, not one %s on its record's key", ix, MoversIndex) } } @@ -555,7 +644,7 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { } got, err := rs.RecordOf(r.Instance, "") if err != nil || len(got) != 1 || got[0].Record.Milestone != recordMilestone("main", 47) || got[0].Waits != "waiting for mesh-x on ace" || - got[0].Bytes == 0 || got[0].Of != 256<<10 { + got[0].Bytes == 0 || got[0].Of != EntryLimit { t.Fatalf("%+v %v", got, err) } if byMilestone, err := rs.RecordOf("", recordMilestone("main", 47)); err != nil || len(byMilestone) != 1 { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go index 5546b825f..7b90a3d5a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -137,6 +137,7 @@ func (rs *Records) end(r *Record, why, by string) error { return err } r.Ended, r.Sequence = &e, seq + rs.clearTooLarge(r) rs.keep(r) if rs.Changed != nil { rs.Changed(r, RecordChange{What: "ended", Why: why, Ended: true}) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go index be806d965..af495c710 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordpipeline.go @@ -186,6 +186,9 @@ func (h *Holder) pipelineStarted(p runner.PipelineStartedEvent) error { if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil { return err } + if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil { + return err + } } return nil } @@ -206,6 +209,9 @@ func (h *Holder) pipelineEnded(p runner.PipelineEndedEvent) error { if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil { return err } + if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil { + return err + } for _, s := range m.Steps { if s.Step == "" || s.Outcome == "" { continue diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 3437a9907..6b39b2cbc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -15,12 +15,9 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// valueLimit is the bus's limit of one value, which a record's state is held under (design 47: a record of 30 modules on -// four nodes is about 30 KiB of it). -const valueLimit = 256 << 10 - -// recordView is one record as gate_record answers it: its state, the sequence of its last operation, its size against -// the value limit, and while open why it has not ended. +// recordView is one record as gate_record answers it, joined from its head entry and its movers' entries (design 47 H2 +// §809): its state, the sequence of its head's last operation, its head entry's size against the 64 KiB an entry holds, +// and while open why it has not ended. type recordView struct { Instance string `json:"instance"` Sequence uint64 `json:"sequence"` @@ -31,8 +28,8 @@ type recordView struct { } func viewOf(r *Record) recordView { - raw, _ := json.Marshal(r) - v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: valueLimit, Record: r} + raw, _ := json.Marshal(r.head()) + v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: EntryLimit, Record: r} if r.Ended == nil { v.Waits = allRun(r) } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 94b485b45..b702a5e67 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -36,7 +36,7 @@ "storage": "ledger", "template": "ledger", "options": { - "max-mib": 768 + "max-mib": 1024 }, "types": [ { @@ -125,6 +125,9 @@ }, "owed": { "type": "object" + }, + "movers": { + "type": "integer" } }, "operations": [ @@ -160,34 +163,11 @@ "sequence": true }, { - "name": "entry", - "effect": "set-entry", + "name": "movers", + "effect": "set-fields", "fields": [ - "key", - "module", - "fingerprint", - "node" + "movers" ], - "into": "entries", - "key": "key", - "sequence": true - }, - { - "name": "outcome", - "effect": "set-entry", - "fields": [ - "key", - "module", - "fingerprint", - "node", - "step", - "outcome", - "why", - "at", - "event" - ], - "into": "outcomes", - "key": "key", "sequence": true }, { @@ -234,6 +214,99 @@ } ] }, + { + "name": "mover", + "schema": { + "record": { + "type": "string" + }, + "module": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "version": { + "type": "string" + }, + "key": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "node": { + "type": "string" + }, + "step": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "why": { + "type": "string" + }, + "at": { + "type": "string" + }, + "event": { + "type": "string" + } + }, + "operations": [ + { + "name": "mover", + "effect": "set-fields", + "fields": [ + "record", + "module", + "pipeline", + "version" + ], + "sequence": true + }, + { + "name": "entry", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node" + ], + "into": "entries", + "key": "key", + "sequence": true + }, + { + "name": "outcome", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node", + "step", + "outcome", + "why", + "at", + "event" + ], + "into": "outcomes", + "key": "key", + "sequence": true + } + ], + "indexes": [ + { + "name": "movers", + "on": [ + "record" + ] + } + ] + }, { "name": "held", "schema": {