GT1: the gate's ledger keeps one entry per module and revision
A record's head entry record/<branch.source> keeps the record-wide fields and the count of its movers; each mover, the catalog half's too, has its entry mover/<branch.source.module> with its pipeline, version and step outcomes per node, found through the index movers on its record's key. An entry over 64 KiB is refused with record-too-large, cleared when written within it or at the record's end; the ledger is capped at 1 GiB. Builds: design 47 H2 §809, §820; H-A-385, H-A-391
This commit is contained in:
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -28,5 +29,35 @@ type FindingWriter interface {
|
||||
// generated contract of the seat that defines it (A-17). It is started once the runtime serves (A-40).
|
||||
func NewFindingWriter() (*views.HeldWriter[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent], error) {
|
||||
return views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{
|
||||
Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken}})
|
||||
Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{HeadUntaken, RecordTooLarge}})
|
||||
}
|
||||
|
||||
// recordTooLarge raises record-too-large for a delivery record's entry the gate refused over EntryLimit, or clears it when
|
||||
// the writer holds it (design 47 H2 §809): the record's records hook.
|
||||
func (h *Holder) recordTooLarge(instance string, bytes int, raised bool) {
|
||||
if h.Findings == nil {
|
||||
return
|
||||
}
|
||||
held := false
|
||||
for _, f := range h.Findings.Held() {
|
||||
if f.Scope == FindingScope && f.ID == instance && f.Kind == RecordTooLarge {
|
||||
held = true
|
||||
}
|
||||
}
|
||||
var err error
|
||||
switch {
|
||||
case raised && !held:
|
||||
now := h.Now().UTC()
|
||||
err = h.Findings.Write(context.Background(), meshconditions.Finding{Scope: FindingScope, ID: instance, Kind: RecordTooLarge,
|
||||
Observer: Module, Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Since: now, At: now,
|
||||
Summary: fmt.Sprintf("%s would hold %d bytes, over the %d one entry holds: the write is refused", instance, bytes, EntryLimit),
|
||||
Headline: "A delivery record's entry is too large", Subject: meshconditions.Subject{Scope: FindingScope, ID: instance},
|
||||
Explanation: "The gate keeps one entry per record and one per mover, each within 64 KiB (design 47 H2 §809); " +
|
||||
"this entry would pass it, so the write is refused. It clears when the entry is next written within 64 KiB or the record ends."})
|
||||
case !raised && held:
|
||||
err = h.Findings.Clear(context.Background(), FindingScope, instance, RecordTooLarge, "the entry is written within 64 KiB, or its record ended")
|
||||
}
|
||||
if err != nil && h.Logf != nil {
|
||||
h.Logf("[mesh-delivery] record-too-large of %s is not said: %v", instance, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ func main() {
|
||||
}
|
||||
h.init()
|
||||
h.SaySeat = stdio.SeatPublish
|
||||
h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf}
|
||||
h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf, TooLarge: h.recordTooLarge}
|
||||
h.Records.Changed = h.recordChanged
|
||||
h.Forge = seatGit{ask: stdio.Ask}
|
||||
h.Catalog = seatCatalog{ask: stdio.Ask}
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
// A record's entries, one per module (design 47 H2 §809; H-A-385, H-A-391): the head entry `record/<branch.source>` keeps
|
||||
// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module; each mover, the
|
||||
// catalog half's too, has its own entry `mover/<branch.source.module>` with its pipeline, version and the module's step
|
||||
// outcomes per node (its machines and their running times among them). No entry names two modules. A record is joined
|
||||
// from its head and the movers the ledger's index `movers` names under the record's key, one query. The gate refuses an
|
||||
// entry over 64 KiB with the finding record-too-large, cleared when that entry is next written within 64 KiB or at the
|
||||
// record's end.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// TypeMover is the ledger type of a mover's entry.
|
||||
const TypeMover = "mover"
|
||||
|
||||
// MoversIndex is the ledger's index of the movers' entries on their record's key (manifest `requires` records' indexes).
|
||||
const MoversIndex = "movers"
|
||||
|
||||
// The operations of a mover's entry; the manifest's type mover declares exactly these (record_test.go).
|
||||
const (
|
||||
OpMover = "mover" // set-fields: its record's key, its module, its pipeline and its version
|
||||
// OpEntry and OpOutcome write its entries and outcomes as they did on the whole record.
|
||||
)
|
||||
|
||||
// MoverOperations are a mover's operations, in the order the manifest's type allows them.
|
||||
var MoverOperations = []string{OpMover, OpEntry, OpOutcome}
|
||||
|
||||
// OpMovers is the head entry's count of its movers (set-fields movers).
|
||||
const OpMovers = "movers"
|
||||
|
||||
// EntryLimit is the most one entry of the ledger holds (§809).
|
||||
const EntryLimit = 64 << 10
|
||||
|
||||
// RecordTooLarge is the finding of an entry refused over EntryLimit (scope delivery).
|
||||
const RecordTooLarge = "record-too-large"
|
||||
|
||||
// Mover is one mover's entry, as apply derives it from its operations.
|
||||
type Mover struct {
|
||||
// Instance and Sequence are its path in the ledger and its last operation; not in the state.
|
||||
Instance string `json:"-"`
|
||||
Sequence uint64 `json:"-"`
|
||||
|
||||
// Record is its record's key, `<branch>.<source>`, the index's field.
|
||||
Record string `json:"record"`
|
||||
Module string `json:"module"`
|
||||
Pipeline string `json:"pipeline,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Entries map[string]RecordEntry `json:"entries,omitempty"`
|
||||
Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"`
|
||||
}
|
||||
|
||||
// Running are the machines the mover runs healthy on, each with when it was first said.
|
||||
func (m *Mover) Running() map[string]time.Time {
|
||||
out := map[string]time.Time{}
|
||||
for _, o := range m.Outcomes {
|
||||
if o.Outcome == OutcomeRunning && o.Node != "" {
|
||||
if at, ok := out[o.Node]; !ok || o.At.Before(at) {
|
||||
out[o.Node] = o.At
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MoverInstance is the entry of a record's mover: `mover/<branch.source.module>`, each escaped as a name.
|
||||
func MoverInstance(milestone, module string) (string, error) {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
return "", fmt.Errorf("a mover of the record %s names its module", milestone)
|
||||
}
|
||||
inst := TypeMover + "/" + milestoneToken(milestone+"."+module)
|
||||
if !segmentRe.MatchString(strings.TrimPrefix(inst, TypeMover+"/")) {
|
||||
return "", fmt.Errorf("the mover %s of %s cannot be named: %q", module, milestone, inst)
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// moversKey is the index key that names every mover of a record: its key's token, then the rest.
|
||||
func moversKey(milestone string) string {
|
||||
return strings.TrimSuffix(buscalls.IndexKey([]string{milestone}, "x"), ".x") + ".>"
|
||||
}
|
||||
|
||||
// movers reads the movers' entries of a record through the index, one query, and each entry.
|
||||
func (rs *Records) movers(milestone string) ([]*Mover, error) {
|
||||
insts, err := rs.Ledger.Query(MoversIndex, moversKey(milestone))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the movers of the record %s cannot be read: %w", milestone, err)
|
||||
}
|
||||
var out []*Mover
|
||||
for _, inst := range insts {
|
||||
raw, seq, err := rs.Ledger.Get(inst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
var m Mover
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("the mover %s is not readable: %w", inst, err)
|
||||
}
|
||||
if m.Record != milestone || m.Module == "" {
|
||||
continue
|
||||
}
|
||||
m.Instance, m.Sequence = inst, seq
|
||||
out = append(out, &m)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// join lays a record's movers into it: their entries and outcomes, by module.
|
||||
func (r *Record) join(ms []*Mover) {
|
||||
r.movers = map[string]*Mover{}
|
||||
for _, m := range ms {
|
||||
r.movers[m.Module] = m
|
||||
for k, e := range m.Entries {
|
||||
if r.Entries == nil {
|
||||
r.Entries = map[string]RecordEntry{}
|
||||
}
|
||||
r.Entries[k] = e
|
||||
}
|
||||
for k, o := range m.Outcomes {
|
||||
if r.Outcomes == nil {
|
||||
r.Outcomes = map[string]RecordOutcome{}
|
||||
}
|
||||
r.Outcomes[k] = o
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Movers are the record's movers' entries, by module.
|
||||
func (r *Record) Movers() map[string]*Mover { return r.movers }
|
||||
|
||||
// mover is the record's entry of module, opened in the ledger when it has none, and counted on the head entry.
|
||||
func (rs *Records) mover(r *Record, module string) (*Mover, error) {
|
||||
if m := r.movers[module]; m != nil {
|
||||
return m, nil
|
||||
}
|
||||
inst, err := MoverInstance(r.Milestone, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := &Mover{Instance: inst, Record: r.Milestone, Module: module}
|
||||
seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Sequence = seq
|
||||
if r.movers == nil {
|
||||
r.movers = map[string]*Mover{}
|
||||
}
|
||||
r.movers[module] = m
|
||||
hseq, err := rs.Ledger.Apply(r.Instance, OpMovers, map[string]any{"movers": len(r.movers)})
|
||||
if err != nil {
|
||||
return m, fmt.Errorf("the mover %s is written and the record %s's count of its movers is not: %w", module, r.Instance, err)
|
||||
}
|
||||
r.MoverCount, r.Sequence = len(r.movers), hseq
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// SetMover writes a mover's pipeline and version, unless its entry holds them already.
|
||||
func (rs *Records) SetMover(r *Record, module, pipeline, version string) error {
|
||||
m, err := rs.mover(r, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := *m
|
||||
next.Pipeline, next.Version = firstOf(pipeline, m.Pipeline), firstOf(version, m.Version)
|
||||
if next.Pipeline == m.Pipeline && next.Version == m.Version {
|
||||
return nil
|
||||
}
|
||||
if err := rs.fits(m.Instance, next); err != nil {
|
||||
return err
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module,
|
||||
"pipeline": next.Pipeline, "version": next.Version})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.Pipeline, m.Version, m.Sequence = next.Pipeline, next.Version, seq
|
||||
rs.fitted(m.Instance)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fits refuses an entry whose state would pass EntryLimit, raising record-too-large for it (§809).
|
||||
func (rs *Records) fits(instance string, state any) error {
|
||||
raw, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(raw) <= EntryLimit {
|
||||
return nil
|
||||
}
|
||||
if rs.TooLarge != nil {
|
||||
rs.TooLarge(instance, len(raw), true)
|
||||
}
|
||||
return fmt.Errorf("the delivery record's entry %s would hold %d bytes, over the %d an entry holds: refused (%s)", instance, len(raw), EntryLimit, RecordTooLarge)
|
||||
}
|
||||
|
||||
// fitted clears record-too-large for an entry written within EntryLimit (TooLarge clears only one it holds).
|
||||
func (rs *Records) fitted(instance string) {
|
||||
if rs.TooLarge != nil {
|
||||
rs.TooLarge(instance, 0, false)
|
||||
}
|
||||
}
|
||||
|
||||
// clearTooLarge clears record-too-large for every entry of a record at its end.
|
||||
func (rs *Records) clearTooLarge(r *Record) {
|
||||
rs.fitted(r.Instance)
|
||||
for _, m := range r.movers {
|
||||
rs.fitted(m.Instance)
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,7 @@ const RecordsLedger = "records"
|
||||
|
||||
// RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate
|
||||
// changes how it writes them.
|
||||
const RecordForm = 1
|
||||
const RecordForm = 2
|
||||
|
||||
// The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it,
|
||||
// since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256
|
||||
@@ -47,8 +47,8 @@ const (
|
||||
OpOpen = "open" // set-fields: the milestone, its branch, source revision and merge call, the heads, when, the form
|
||||
OpGroup = "group" // set-fields: the group's state
|
||||
OpTrace = "trace" // set-fields: the trace
|
||||
OpEntry = "entry" // set-entry into entries: one entry of the graph difference, by its module, fingerprint and node
|
||||
OpOutcome = "outcome" // set-entry into outcomes, by the entry, the node and the step
|
||||
OpEntry = "entry" // a mover's: set-entry into entries, one entry of the graph difference, by its module, fingerprint and node
|
||||
OpOutcome = "outcome" // a mover's: set-entry into outcomes, by the entry, the node and the step
|
||||
OpAway = "away" // set-entry into away, by node
|
||||
OpBack = "back" // remove from away, by node
|
||||
OpEnd = "end" // set-fields: ended
|
||||
@@ -58,7 +58,7 @@ const (
|
||||
)
|
||||
|
||||
// RecordOperations are a record's operations, in the order the manifest's type allows them.
|
||||
var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpAct, OpEnd}
|
||||
var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpAway, OpBack, OpAct, OpEnd}
|
||||
|
||||
// HeldOperations are the month's list's operations.
|
||||
var HeldOperations = []string{OpHold, OpDrop}
|
||||
@@ -99,6 +99,11 @@ type Record struct {
|
||||
Ended *RecordEnded `json:"ended,omitempty"`
|
||||
// Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23).
|
||||
Recovery *RecordRecovery `json:"recovery,omitempty"`
|
||||
// MoverCount is the count of its movers, each with an entry of its own (design 47 H2 §809); Entries and Outcomes above
|
||||
// are joined from those entries when read.
|
||||
MoverCount int `json:"movers,omitempty"`
|
||||
|
||||
movers map[string]*Mover
|
||||
}
|
||||
|
||||
// RecordRecovery is the field recovery of a record: who named the merge a recovery (the caller the runtime proved), why,
|
||||
@@ -267,6 +272,8 @@ type RecordLedger interface {
|
||||
Get(instance string) (json.RawMessage, uint64, error)
|
||||
// Remove removes a record with its operations and its snapshot through the tool runner's ledger removal.
|
||||
Remove(instance string) error
|
||||
// Query is the instances a declared index names under key (`>` for the rest).
|
||||
Query(index, key string) ([]string, error)
|
||||
}
|
||||
|
||||
// sdkLedger is the gate's ledger through the runtime: the tool runner's ledger calls, on the gate's own ledger only
|
||||
@@ -316,6 +323,22 @@ func (sdkLedger) Remove(instance string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdkLedger) Query(index, key string) ([]string, error) {
|
||||
out, err := ledgerCall(buscalls.LedgerQuery, buscalls.QueryRequest{Ledger: RecordsLedger, Index: index, Key: key})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var got buscalls.QueryAnswer
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
return nil, fmt.Errorf("the ledger's answer to the query of %s is not readable: %w", index, err)
|
||||
}
|
||||
insts := make([]string, 0, len(got.Entries))
|
||||
for _, e := range got.Entries {
|
||||
insts = append(insts, e.Instance)
|
||||
}
|
||||
return insts, nil
|
||||
}
|
||||
|
||||
func (sdkLedger) Get(instance string) (json.RawMessage, uint64, error) {
|
||||
out, err := ledgerCall(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: RecordsLedger, Instance: instance})
|
||||
if err != nil {
|
||||
@@ -345,6 +368,8 @@ type Records struct {
|
||||
// Changed is told of every change of a record that owes the forge an act (A-39): a step's outcome applied, the record
|
||||
// ended (owedacts.go); nil owes none.
|
||||
Changed func(r *Record, c RecordChange)
|
||||
// TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none.
|
||||
TooLarge func(instance string, bytes int, raised bool)
|
||||
|
||||
// held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the
|
||||
// gate is their one writer.
|
||||
@@ -454,6 +479,11 @@ func (rs *Records) Read(instance string) (*Record, error) {
|
||||
return nil, nil
|
||||
}
|
||||
r.Instance, r.Sequence = instance, seq
|
||||
ms, err := rs.movers(r.Milestone)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.join(ms)
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
@@ -628,6 +658,11 @@ func (rs *Records) Sweep(now time.Time) error {
|
||||
return err
|
||||
}
|
||||
if r != nil {
|
||||
for _, m := range r.movers {
|
||||
if err := rs.Ledger.Remove(m.Instance); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := rs.Ledger.Remove(e.Instance); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -646,11 +681,17 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error {
|
||||
if r.Group != nil && reflect.DeepEqual(*r.Group, g) {
|
||||
return nil
|
||||
}
|
||||
next := *r
|
||||
next.Group = &g
|
||||
if err := rs.fits(r.Instance, next.head()); err != nil {
|
||||
return err
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Group, r.Sequence = &g, seq
|
||||
rs.fitted(r.Instance)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -659,11 +700,17 @@ func (rs *Records) SetTrace(r *Record, t RecordTrace) error {
|
||||
if r.Trace != nil && reflect.DeepEqual(*r.Trace, t) {
|
||||
return nil
|
||||
}
|
||||
next := *r
|
||||
next.Trace = &t
|
||||
if err := rs.fits(r.Instance, next.head()); err != nil {
|
||||
return err
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Trace, r.Sequence = &t, seq
|
||||
rs.fitted(r.Instance)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -677,14 +724,25 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error)
|
||||
if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) {
|
||||
return false, nil
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(r.Instance, OpOutcome, o)
|
||||
m, err := rs.mover(r, o.Module)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
next := *m
|
||||
next.Outcomes = withEntry(m.Outcomes, o.Key, o)
|
||||
if err := rs.fits(m.Instance, next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(m.Instance, OpOutcome, o)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
m.Outcomes, m.Sequence = next.Outcomes, seq
|
||||
rs.fitted(m.Instance)
|
||||
if r.Outcomes == nil {
|
||||
r.Outcomes = map[string]RecordOutcome{}
|
||||
}
|
||||
r.Outcomes[o.Key], r.Sequence = o, seq
|
||||
r.Outcomes[o.Key] = o
|
||||
if rs.Changed != nil {
|
||||
rs.Changed(r, RecordChange{What: outcomeWords(o), Why: firstOf(o.Why, "the "+o.Step+" step's event")})
|
||||
}
|
||||
@@ -697,17 +755,44 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error {
|
||||
if had, ok := r.Entries[e.Key]; ok && had == e {
|
||||
return nil
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(r.Instance, OpEntry, e)
|
||||
m, err := rs.mover(r, e.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := *m
|
||||
next.Entries = withEntry(m.Entries, e.Key, e)
|
||||
if err := rs.fits(m.Instance, next); err != nil {
|
||||
return err
|
||||
}
|
||||
seq, err := rs.Ledger.Apply(m.Instance, OpEntry, e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.Entries, m.Sequence = next.Entries, seq
|
||||
rs.fitted(m.Instance)
|
||||
if r.Entries == nil {
|
||||
r.Entries = map[string]RecordEntry{}
|
||||
}
|
||||
r.Entries[e.Key], r.Sequence = e, seq
|
||||
r.Entries[e.Key] = e
|
||||
return nil
|
||||
}
|
||||
|
||||
// withEntry is a copy of m with v under k.
|
||||
func withEntry[V any](m map[string]V, k string, v V) map[string]V {
|
||||
out := make(map[string]V, len(m)+1)
|
||||
for key, val := range m {
|
||||
out[key] = val
|
||||
}
|
||||
out[k] = v
|
||||
return out
|
||||
}
|
||||
|
||||
// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's.
|
||||
func (r Record) head() Record {
|
||||
r.Entries, r.Outcomes, r.movers = nil, nil, nil
|
||||
return r
|
||||
}
|
||||
|
||||
// sameOutcome is whether two outcomes say the same, whenever and from whichever event they were taken.
|
||||
func sameOutcome(a, b RecordOutcome) bool {
|
||||
a.At, b.At, a.Event, b.Event = time.Time{}, time.Time{}, "", ""
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -150,6 +151,33 @@ func (f *ledgerFake) Remove(instance string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Query answers a declared index as the tool runner's watcher keeps it: the key of each instance whose type declares the
|
||||
// index, from the values of its fields and its path (buscalls.IndexKey), matched with `>` for the rest.
|
||||
func (f *ledgerFake) Query(index, key string) ([]string, error) {
|
||||
var out []string
|
||||
for inst, raw := range f.states {
|
||||
ty := f.types[strings.SplitN(inst, "/", 2)[0]]
|
||||
for _, ix := range ty.Indexes {
|
||||
if ix.Name != index || string(raw) == "null" {
|
||||
continue
|
||||
}
|
||||
state := map[string]any{}
|
||||
_ = json.Unmarshal(raw, &state)
|
||||
var vals []string
|
||||
for _, on := range ix.On {
|
||||
v, _ := state[on].(string)
|
||||
vals = append(vals, v)
|
||||
}
|
||||
k := buscalls.IndexKey(vals, inst)
|
||||
if k == key || strings.HasSuffix(key, ">") && strings.HasPrefix(k, strings.TrimSuffix(key, ">")) {
|
||||
out = append(out, inst)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) {
|
||||
s := f.states[instance]
|
||||
if s == nil || string(s) == "null" {
|
||||
@@ -165,6 +193,64 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T
|
||||
same := map[string]string{"novox/lib@4": strings.Repeat("2", 40), "novox/app@3": strings.Repeat("1", 40)}
|
||||
at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC)
|
||||
|
||||
// One entry per module and revision (design 47 H2 §809; H-A-385, H-A-391): 126 movers write 127 entries, none over
|
||||
// 64 KiB, the head counting 126; an entry made too large is refused with record-too-large, cleared at the record's end.
|
||||
t.Run("a record of 126 movers writes 127 entries, each within 64 KiB", func(t *testing.T) {
|
||||
l := newLedgerFake(t)
|
||||
found := map[string]bool{}
|
||||
rs := &Records{Ledger: l, Now: func() time.Time { return at }, TooLarge: func(inst string, _ int, raised bool) {
|
||||
if raised {
|
||||
found[inst] = true
|
||||
} else {
|
||||
delete(found, inst)
|
||||
}
|
||||
}}
|
||||
r, _, err := rs.Open(opening("main", 90, heads))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range 126 {
|
||||
module := fmt.Sprintf("mod-%03d", i)
|
||||
if err := rs.AddEntry(r, RecordEntry{Module: module, Fingerprint: "src2:f" + module}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rs.SetMover(r, module, "main.90", "0.1.7"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"ace", "g14", "novox", "shanks"} {
|
||||
if _, err := rs.Outcome(r, RecordOutcome{Module: module, Fingerprint: "src2:f" + module, Node: node, Step: "running",
|
||||
Outcome: OutcomeRunning, At: at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
entries := 0
|
||||
for inst, raw := range l.states {
|
||||
if strings.HasPrefix(inst, TypeRecord+"/") || strings.HasPrefix(inst, TypeMover+"/") {
|
||||
entries++
|
||||
if len(raw) > EntryLimit {
|
||||
t.Errorf("%s holds %d bytes", inst, len(raw))
|
||||
}
|
||||
if strings.HasPrefix(inst, TypeMover+"/") && strings.Count(string(raw), `"module":"mod-`) != 1+1+4 {
|
||||
t.Errorf("%s names another module: %s", inst, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
read, err := rs.Read(r.Instance)
|
||||
if err != nil || entries != 127 || read.MoverCount != 126 || len(read.Movers()) != 126 || len(read.Outcomes) != 504 ||
|
||||
len(read.Movers()["mod-007"].Running()) != 4 || read.Movers()["mod-007"].Version != "0.1.7" {
|
||||
t.Fatalf("%d entries, the head counts %d, %d joined, %d outcomes, %v", entries, read.MoverCount, len(read.Movers()), len(read.Outcomes), err)
|
||||
}
|
||||
why := strings.Repeat("w", EntryLimit)
|
||||
if _, err := rs.Outcome(read, RecordOutcome{Module: "mod-001", Fingerprint: "src2:fmod-001", Node: "ace", Step: "build",
|
||||
Outcome: OutcomeFailed, Why: why, At: at}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 1 {
|
||||
t.Fatalf("an entry over 64 KiB was taken (%v), findings %v", err, found)
|
||||
}
|
||||
if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 {
|
||||
t.Fatalf("the record's end left %v (%v)", found, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an instance per source revision, a path the tool runner takes", func(t *testing.T) {
|
||||
a, err := RecordInstance(recordMilestone("feat/x", 7))
|
||||
if err != nil {
|
||||
@@ -322,7 +408,7 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T
|
||||
// W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB.
|
||||
func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) {
|
||||
l := newLedgerFake(t)
|
||||
for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations} {
|
||||
for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations} {
|
||||
var got []string
|
||||
for _, op := range l.types[name].Operations {
|
||||
got = append(got, op.Name)
|
||||
@@ -332,8 +418,11 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) {
|
||||
}
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if !strings.Contains(string(raw), `"max-mib": 768`) {
|
||||
t.Error("the records ledger is not capped at 768 MiB")
|
||||
if !strings.Contains(string(raw), `"max-mib": 1024`) {
|
||||
t.Error("the records ledger is not capped at 1 GiB (design 47 H2 §809)")
|
||||
}
|
||||
if ix := l.types[TypeMover].Indexes; len(ix) != 1 || ix[0].Name != MoversIndex || !slices.Equal(ix[0].On, []string{"record"}) {
|
||||
t.Errorf("the type mover's indexes are %+v, not one %s on its record's key", ix, MoversIndex)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -555,7 +644,7 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) {
|
||||
}
|
||||
got, err := rs.RecordOf(r.Instance, "")
|
||||
if err != nil || len(got) != 1 || got[0].Record.Milestone != recordMilestone("main", 47) || got[0].Waits != "waiting for mesh-x on ace" ||
|
||||
got[0].Bytes == 0 || got[0].Of != 256<<10 {
|
||||
got[0].Bytes == 0 || got[0].Of != EntryLimit {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
if byMilestone, err := rs.RecordOf("", recordMilestone("main", 47)); err != nil || len(byMilestone) != 1 {
|
||||
|
||||
@@ -137,6 +137,7 @@ func (rs *Records) end(r *Record, why, by string) error {
|
||||
return err
|
||||
}
|
||||
r.Ended, r.Sequence = &e, seq
|
||||
rs.clearTooLarge(r)
|
||||
rs.keep(r)
|
||||
if rs.Changed != nil {
|
||||
rs.Changed(r, RecordChange{What: "ended", Why: why, Ended: true})
|
||||
|
||||
@@ -186,6 +186,9 @@ func (h *Holder) pipelineStarted(p runner.PipelineStartedEvent) error {
|
||||
if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -206,6 +209,9 @@ func (h *Holder) pipelineEnded(p runner.PipelineEndedEvent) error {
|
||||
if err := h.Records.AddEntry(r, RecordEntry{Module: m.Module, Fingerprint: m.Fingerprint}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.Records.SetMover(r, m.Module, p.Pipeline, m.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, s := range m.Steps {
|
||||
if s.Step == "" || s.Outcome == "" {
|
||||
continue
|
||||
|
||||
@@ -15,12 +15,9 @@ import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// valueLimit is the bus's limit of one value, which a record's state is held under (design 47: a record of 30 modules on
|
||||
// four nodes is about 30 KiB of it).
|
||||
const valueLimit = 256 << 10
|
||||
|
||||
// recordView is one record as gate_record answers it: its state, the sequence of its last operation, its size against
|
||||
// the value limit, and while open why it has not ended.
|
||||
// recordView is one record as gate_record answers it, joined from its head entry and its movers' entries (design 47 H2
|
||||
// §809): its state, the sequence of its head's last operation, its head entry's size against the 64 KiB an entry holds,
|
||||
// and while open why it has not ended.
|
||||
type recordView struct {
|
||||
Instance string `json:"instance"`
|
||||
Sequence uint64 `json:"sequence"`
|
||||
@@ -31,8 +28,8 @@ type recordView struct {
|
||||
}
|
||||
|
||||
func viewOf(r *Record) recordView {
|
||||
raw, _ := json.Marshal(r)
|
||||
v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: valueLimit, Record: r}
|
||||
raw, _ := json.Marshal(r.head())
|
||||
v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: EntryLimit, Record: r}
|
||||
if r.Ended == nil {
|
||||
v.Waits = allRun(r)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
"storage": "ledger",
|
||||
"template": "ledger",
|
||||
"options": {
|
||||
"max-mib": 768
|
||||
"max-mib": 1024
|
||||
},
|
||||
"types": [
|
||||
{
|
||||
@@ -125,6 +125,9 @@
|
||||
},
|
||||
"owed": {
|
||||
"type": "object"
|
||||
},
|
||||
"movers": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"operations": [
|
||||
@@ -160,34 +163,11 @@
|
||||
"sequence": true
|
||||
},
|
||||
{
|
||||
"name": "entry",
|
||||
"effect": "set-entry",
|
||||
"name": "movers",
|
||||
"effect": "set-fields",
|
||||
"fields": [
|
||||
"key",
|
||||
"module",
|
||||
"fingerprint",
|
||||
"node"
|
||||
"movers"
|
||||
],
|
||||
"into": "entries",
|
||||
"key": "key",
|
||||
"sequence": true
|
||||
},
|
||||
{
|
||||
"name": "outcome",
|
||||
"effect": "set-entry",
|
||||
"fields": [
|
||||
"key",
|
||||
"module",
|
||||
"fingerprint",
|
||||
"node",
|
||||
"step",
|
||||
"outcome",
|
||||
"why",
|
||||
"at",
|
||||
"event"
|
||||
],
|
||||
"into": "outcomes",
|
||||
"key": "key",
|
||||
"sequence": true
|
||||
},
|
||||
{
|
||||
@@ -234,6 +214,99 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mover",
|
||||
"schema": {
|
||||
"record": {
|
||||
"type": "string"
|
||||
},
|
||||
"module": {
|
||||
"type": "string"
|
||||
},
|
||||
"pipeline": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"fingerprint": {
|
||||
"type": "string"
|
||||
},
|
||||
"node": {
|
||||
"type": "string"
|
||||
},
|
||||
"step": {
|
||||
"type": "string"
|
||||
},
|
||||
"outcome": {
|
||||
"type": "string"
|
||||
},
|
||||
"why": {
|
||||
"type": "string"
|
||||
},
|
||||
"at": {
|
||||
"type": "string"
|
||||
},
|
||||
"event": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"operations": [
|
||||
{
|
||||
"name": "mover",
|
||||
"effect": "set-fields",
|
||||
"fields": [
|
||||
"record",
|
||||
"module",
|
||||
"pipeline",
|
||||
"version"
|
||||
],
|
||||
"sequence": true
|
||||
},
|
||||
{
|
||||
"name": "entry",
|
||||
"effect": "set-entry",
|
||||
"fields": [
|
||||
"key",
|
||||
"module",
|
||||
"fingerprint",
|
||||
"node"
|
||||
],
|
||||
"into": "entries",
|
||||
"key": "key",
|
||||
"sequence": true
|
||||
},
|
||||
{
|
||||
"name": "outcome",
|
||||
"effect": "set-entry",
|
||||
"fields": [
|
||||
"key",
|
||||
"module",
|
||||
"fingerprint",
|
||||
"node",
|
||||
"step",
|
||||
"outcome",
|
||||
"why",
|
||||
"at",
|
||||
"event"
|
||||
],
|
||||
"into": "outcomes",
|
||||
"key": "key",
|
||||
"sequence": true
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "movers",
|
||||
"on": [
|
||||
"record"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "held",
|
||||
"schema": {
|
||||
|
||||
Reference in New Issue
Block a user