From 758a8ad6532625f2d439944836ba98240ce0a37c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:58:39 +0100 Subject: [PATCH] GT1: each owed act is a ledger entry of its own; every ledger write is measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An act the gate owes the forge is now the entry act/ (type act, index acts on its record's key in the manifest), removed when the forge bridge performed it; the head entry holds the record's fields alone, and the record's acts are joined from their entries when read. The record type keeps no act operation: a head written before this change is read for its fields and the acts embedded in it are ignored (they are notes and views of the few records in flight at the delivery; losing them changes nothing the operator relies on). A view act's text is the record's summary, the head's fields and a count of movers per outcome, so it fits 64 KiB for any record; gate_record still shows the whole record. The 64 KiB refusal measures every write, the head's, a mover's and an act's, raising record-too-large, which the record's end clears for every entry of it. The gate's whole test gains the step: 126 movers write 127 entries plus their acts, none over 64 KiB, the head counting 126 and holding no act; a take-in said again with a why past 64 KiB raises record-too-large and the record's end clears it. Builds: design 47 H2 §809, H-A-385, H-A-391, H-A-453 (3), H-A-455, H-A-457, H-A-459 --- .../cmd/mesh-delivery-gate/bus_test.go | 10 +- .../mesh-delivery-gate/mergecall_bus_test.go | 70 ++++++++++++ .../cmd/mesh-delivery-gate/movers.go | 78 ++++++++++---- .../cmd/mesh-delivery-gate/owedacts.go | 38 ++++++- .../cmd/mesh-delivery-gate/owedacts_test.go | 40 ++++++- .../cmd/mesh-delivery-gate/owedended.go | 78 +++++++++++++- .../cmd/mesh-delivery-gate/record.go | 55 +++++----- .../cmd/mesh-delivery-gate/record_test.go | 11 +- .../cmd/mesh-delivery-gate/recordevents.go | 23 +++- .../cmd/mesh-delivery-gate/recovery.go | 4 +- .../cmd/mesh-delivery-gate/times.go | 6 +- modules/mesh-delivery-gate/module.json | 101 +++++++++++------- 12 files changed, 406 insertions(+), 108 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index a6af04418..e525e2424 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -395,7 +395,7 @@ func (r *runtime) answer(m rpcMessage) { return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) } switch m.Method { - case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery: + case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery, buscalls.LedgerRemove: // The gate's ledger `records` (W11.1), as the tool runner keeps it: the record test's ledger in memory. r.b.mu.Lock() defer r.b.mu.Unlock() @@ -414,6 +414,14 @@ func (r *runtime) answer(m rpcMessage) { } return a, err } + if m.Method == buscalls.LedgerRemove { + // An act performed is removed (design 47 H2 §809, H-A-455 (1)), as the tool runner's ledger removal does. + var in buscalls.LedgerRemoveRequest + if err := json.Unmarshal(m.Params, &in); err != nil { + return nil, err + } + return buscalls.LedgerRemoved{}, r.b.ledger.Remove(in.Instance) + } if m.Method == buscalls.LedgerGet { var in buscalls.InstanceRequest if err := json.Unmarshal(m.Params, &in); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go index 62f9e33bb..c21674dd2 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "testing" "time" @@ -110,6 +111,75 @@ func TestAGateWholeTakesAMergeCallAndWatchesTheHeadsOverARealBus(t *testing.T) { want.Round(time.Minute)) } + // GT1 (design 47 H2 §809; H-A-385, H-A-391, H-A-453 (3), H-A-455): a record of 126 movers writes 127 entries, its head + // and one per mover, and its acts owed the forge each an entry of its own, none over 64 KiB, the head counting 126 and + // holding no act; an entry made too large raises record-too-large, and the record's end clears it. + call2, merge2 := "00000000000000c2", strings.Repeat("e", 40) + instance2, _ := RecordInstance(recordMilestone(branch, 2)) + b.publish(t, "mesh.seat.git.event.merge-called."+call2, map[string]any{"call": call2, "by": "lead@shanks", "account": "acct-s", + "at": time.Now().UTC().Format(time.RFC3339), "members": []any{map[string]any{"repository": "novox/app", "number": 6, + "branch": branch, "head": strings.Repeat("f", 40), "commit": merge2}}}) + rt.until("the record of the second merge call", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.ledger.states[instance2] != nil + }) + var taken []any + for i := range 126 { + module := "mesh-module-" + strings.Repeat("x", 3) + fmt.Sprintf("%03d", i) + taken = append(taken, map[string]any{"module": module, "commit": merge2, "fingerprint": fmt.Sprintf("%064x", i), "repository": "novox/app"}) + } + b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 2, "revision": 2, + "call": call2, "entries": taken}) + rt.until("the record's 126 movers taken in", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + last, _ := MoverInstance(recordMilestone(branch, 2), taken[125].(map[string]any)["module"].(string)) + return strings.Contains(string(b.ledger.states[instance2]), `"movers":126`) && + strings.Contains(string(b.ledger.states[last]), `"outcome":"`+OutcomeTakenIn+`"`) + }) + b.mu.Lock() + entries, acts, over := 0, 0, []string{} + for inst, raw := range b.ledger.states { + if string(raw) == "null" { + continue + } + var of struct { + Milestone string `json:"milestone"` + Record string `json:"record"` + } + _ = json.Unmarshal(raw, &of) + if of.Milestone != recordMilestone(branch, 2) && of.Record != recordMilestone(branch, 2) { + continue + } + if strings.HasPrefix(inst, TypeAct+"/") { + acts++ + } else { + entries++ + } + if len(raw) > EntryLimit { + over = append(over, fmt.Sprintf("%s holds %d bytes", inst, len(raw))) + } + } + headRaw := string(b.ledger.states[instance2]) + b.mu.Unlock() + if entries != 127 || acts == 0 || len(over) > 0 || strings.Contains(headRaw, `"acts"`) { + t.Fatalf("the record of 126 movers writes %d entries and %d acts, over 64 KiB: %v; its head: %.300s", entries, acts, over, headRaw) + } + if findingHeld(ctx, b, RecordTooLarge, "") { + t.Fatal("record-too-large is raised for a record whose entries are within 64 KiB") + } + // The first mover's take-in said again with a why past 64 KiB: its entry's write is refused and raises record-too-large; + // the record's end clears it. + first := taken[0].(map[string]any) + tooLarge, _ := MoverInstance(recordMilestone(branch, 2), first["module"].(string)) + b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 2, "revision": 2, + "call": call2, "why": strings.Repeat("w", EntryLimit), "entries": []any{first}}) + rt.until("record-too-large raised for the mover's entry", eventWithin, func() bool { return findingHeld(ctx, b, RecordTooLarge, tooLarge) }) + b.publish(t, "mesh.seat.mesh-wants.event.pipeline-wanted."+escaped+".2", map[string]any{"v": 1, "branch": branch, + "source": 2, "pipeline": recordMilestone(branch, 2), "outcome": "running"}) + rt.until("record-too-large cleared at the record's end", eventWithin, func() bool { return !findingHeld(ctx, b, RecordTooLarge, "") }) + // The seed (§768): every watched branch's first source revision at its head; its head-moved then holds nothing. seeded := strings.Repeat("5", 40) b.mu.Lock() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go index 4fc43b265..f70f710ed 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -1,12 +1,13 @@ package main // A record's entries, one per module (design 47 H2 §809; H-A-385, H-A-391): the head entry `record/` keeps -// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module; each mover, the -// catalog half's too, has its own entry `mover/` with its pipeline, version and the module's step -// outcomes per node (its machines and their running times among them). No entry names two modules. A record is joined -// from its head and the movers the ledger's index `movers` names under the record's key, one query. The gate refuses an -// entry over 64 KiB with the finding record-too-large, cleared when that entry is next written within 64 KiB or at the -// record's end. +// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module and no act; each +// mover, the catalog half's too, has its own entry `mover/` with its pipeline, version and the +// module's step outcomes per node (its machines and their running times among them); each act owed the forge has its own +// entry `act/`, removed when performed (H-A-455 (1)). No entry names two modules. A record is joined +// from its head and the movers and acts the ledger's indexes `movers` and `acts` name under the record's key, one query +// each. The gate refuses a write over 64 KiB to any entry, the head's, a mover's or an act's, with the finding +// record-too-large, cleared when that entry is next written within 64 KiB or at the record's end (H-A-455 (3)). import ( "encoding/json" @@ -85,7 +86,7 @@ func MoverInstance(milestone, module string) (string, error) { return inst, nil } -// moversKey is the index key that names every mover of a record: its key's token, then the rest. +// moversKey is the index key that names every mover or act of a record: its key's token, then the rest. func moversKey(milestone string) string { return strings.TrimSuffix(buscalls.IndexKey([]string{milestone}, "x"), ".x") + ".>" } @@ -160,7 +161,7 @@ func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) class = firstOf(c, catalogue.ClassLeaf) } m := &Mover{Instance: inst, Record: r.Milestone, Module: module, Class: class} - seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}) + seq, err := rs.write(r, inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}, m) if err != nil { return nil, err } @@ -169,7 +170,9 @@ func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) r.movers = map[string]*Mover{} } r.movers[module] = m - hseq, err := rs.Ledger.Apply(r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}) + next := *r + next.MoverCount = len(r.movers) + hseq, err := rs.write(r, r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}, next.head()) if err != nil { return m, fmt.Errorf("the mover %s is written and the record %s's count of its movers is not: %w", module, r.Instance, err) } @@ -188,21 +191,33 @@ func (rs *Records) SetMover(r *Record, module, fingerprint, pipeline, version st if next.Pipeline == m.Pipeline && next.Version == m.Version { return nil } - if err := rs.fits(m.Instance, next); err != nil { - return err - } - seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, - "pipeline": next.Pipeline, "version": next.Version}) + seq, err := rs.write(r, m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, + "pipeline": next.Pipeline, "version": next.Version}, next) if err != nil { return err } m.Pipeline, m.Version, m.Sequence = next.Pipeline, next.Version, seq - rs.fitted(m.Instance) return nil } -// fits refuses an entry whose state would pass EntryLimit, raising record-too-large for it (§809). -func (rs *Records) fits(instance string, state any) error { +// write is one write to an entry of a record, measured first: the entry's state after it (the head's without its movers +// and acts) is refused over EntryLimit with record-too-large, and a write taken clears the entry's finding (§809, H-A-455 +// (3)). +func (rs *Records) write(r *Record, instance, operation string, body, state any) (uint64, error) { + if err := rs.fits(r, instance, state); err != nil { + return 0, err + } + seq, err := rs.Ledger.Apply(instance, operation, body) + if err != nil { + return 0, err + } + rs.fitted(instance) + return seq, nil +} + +// fits refuses an entry of r whose state would pass EntryLimit, raising record-too-large for it (§809); the entry is +// remembered with its record, so the record's end clears it whichever entry it was (H-A-455 (3)). +func (rs *Records) fits(r *Record, instance string, state any) error { raw, err := json.Marshal(state) if err != nil { return err @@ -210,6 +225,12 @@ func (rs *Records) fits(instance string, state any) error { if len(raw) <= EntryLimit { return nil } + rs.mu.Lock() + if rs.refused == nil { + rs.refused = map[string]string{} + } + rs.refused[instance] = r.Milestone + rs.mu.Unlock() if rs.TooLarge != nil { rs.TooLarge(instance, len(raw), true) } @@ -223,10 +244,27 @@ func (rs *Records) fitted(instance string) { } } -// clearTooLarge clears record-too-large for every entry of a record at its end. +// clearTooLarge clears record-too-large for every entry of a record at its end: its head, its movers, its acts and every +// entry of it a write was refused for. func (rs *Records) clearTooLarge(r *Record) { - rs.fitted(r.Instance) + insts := []string{r.Instance} for _, m := range r.movers { - rs.fitted(m.Instance) + insts = append(insts, m.Instance) + } + for id := range r.Acts { + if inst, err := ActInstance(r.Milestone, id); err == nil { + insts = append(insts, inst) + } + } + rs.mu.Lock() + for inst, of := range rs.refused { + if of == r.Milestone { + insts = append(insts, inst) + delete(rs.refused, inst) + } + } + rs.mu.Unlock() + for _, inst := range insts { + rs.fitted(inst) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go index e27918d7d..b297ff9dc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go @@ -12,9 +12,9 @@ import ( // What the gate owes the forge for a change of a delivery record (design 47 G A-39, A-35, §158, §161 to §167): every change of // a record, a step's outcome applied or the record ended, says a line on the commit's note of each member pull request's head -// (` record : ()`) and replaces each member's delivery view with what `show` answers for the -// record; a comment is said only when the record ends failed or held, naming why. Each act goes through owe (owed.go): kept in -// the record's acts, said as the seat's event owed, and ended on owed.ended. +// (` record : ()`) and replaces each member's delivery view with the record's summary, its head's +// fields and a count of its movers per outcome (H-A-459); a comment is said only when the record ends failed or held, naming why. Each act goes through owe (owed.go): kept as +// an entry of its own among the record's acts, said as the seat's event owed, and ended on owed.ended. // RecordChange is one change of a record: what changed and why; Ended when it is the record's end. type RecordChange struct { @@ -39,7 +39,7 @@ func (h *Holder) recordChanged(r *Record, c RecordChange) { } now := h.Now().UTC() line := fmt.Sprintf("%s record %s: %s (%s)", now.Format("2006-01-02T15:04:05Z"), r.Instance, c.What, c.Why) - view, err := json.MarshalIndent(map[string]any{"records": []recordView{viewOf(r)}}, "", " ") + view, err := json.MarshalIndent(map[string]any{"records": []recordSummary{summaryOf(r)}}, "", " ") if err != nil { h.Refuse(fmt.Errorf("the record %s cannot be shown on its pull requests: %w", r.Instance, err)) return @@ -70,6 +70,36 @@ func (h *Holder) recordChanged(r *Record, c RecordChange) { } } +// recordSummary is the record a view act shows (design 47 H2 §809, H-A-455 (2), H-A-459): the head's fields, without its +// acts, and a count of its movers per outcome, never the movers' entries, so a view act of any record fits 64 KiB; the +// whole record stays readable through gate_record. +type recordSummary struct { + recordView + MoversPerOutcome map[string]int `json:"movers-per-outcome,omitempty"` +} + +// summaryOf is the record a view act shows. +func summaryOf(r *Record) recordSummary { + v := viewOf(r) + head := r.head() + v.Record = &head + per := map[string]map[string]bool{} + for _, o := range r.Outcomes { + if per[o.Outcome] == nil { + per[o.Outcome] = map[string]bool{} + } + per[o.Outcome][o.Module] = true + } + out := recordSummary{recordView: v} + for outcome, movers := range per { + if out.MoversPerOutcome == nil { + out.MoversPerOutcome = map[string]int{} + } + out.MoversPerOutcome[outcome] = len(movers) + } + return out +} + // endedFailedOrHeld is the comment a record ended says when one of its outcomes is failed or held, naming why; "" otherwise. func endedFailedOrHeld(r *Record) string { var lines []string diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go index f7331bc9e..30c7c83b0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "strings" "testing" "time" @@ -41,7 +42,7 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. return } t.Run("an outcome applied says the note and the view, no comment", func(t *testing.T) { - _, rs, r, f := setup(t) + h, rs, r, f := setup(t) if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil { t.Fatal(err) } @@ -62,7 +63,8 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if o.Act == seat.OwedActNote && (o.Line != "2026-11-11T21:00:00Z record "+r.Instance+": mesh-x on ace build built (built it)" || o.Commit != head) { t.Fatalf("note %+v", o) } - if o.Act == seat.OwedActView && !strings.Contains(o.Text, r.Instance) { + // H-A-459: the view is the record's head fields and a count of movers per outcome, never the movers' entries. + if o.Act == seat.OwedActView && (!strings.Contains(o.Text, r.Instance) || !strings.Contains(strings.Join(strings.Fields(o.Text), ""), `"movers-per-outcome":{"built":1}`) || strings.Contains(o.Text, `"outcomes"`) || strings.Contains(o.Text, `"entries"`)) { t.Fatalf("view %q", o.Text) } } @@ -70,6 +72,36 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil || len(f.said) != 2 { t.Fatalf("said again: %v %d", err, len(f.said)) } + // Design 47 H2 §809, H-A-455 (1): each act is an entry of its own, act/, naming its record; the + // head entry holds none, and an act performed is removed. + l := rs.Ledger.(*ledgerFake) + for _, o := range f.said { + inst, err := ActInstance(r.Milestone, o.ID) + if err != nil || !strings.Contains(string(l.states[inst]), `"record":"main.47"`) { + t.Fatalf("the act %s has no entry of its own at %s: %s (%v)", o.ID, inst, l.states[inst], err) + } + } + if strings.Contains(string(l.states[r.Instance]), `"acts"`) { + t.Fatalf("the head entry holds acts: %s", l.states[r.Instance]) + } + note := f.said[0] + body, err := forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: note.ID, Outcome: forgemsg.OwedPerformed, At: at}) + if err != nil { + t.Fatal(err) + } + if err := h.OwedEnded(body); err != nil { + t.Fatal(err) + } + inst, _ := ActInstance(r.Milestone, note.ID) + if string(l.states[inst]) != "null" || h.Status("")["owed"] != 1 { + t.Fatalf("the act performed is kept as %s, %v owed", l.states[inst], h.Status("")["owed"]) + } + // H-A-457: a head written before, its acts embedded, is read for its fields; those acts are ignored. + l.states[r.Instance] = json.RawMessage(strings.Replace(string(l.states[r.Instance]), `{`, `{"acts":{"old":{"id":"old","act":"view"}},`, 1)) + read, err := rs.Read(r.Instance) + if _, old := read.Acts["old"]; err != nil || old || len(read.Acts) != 1 || read.Milestone != "main.47" { + t.Fatalf("an old head's embedded acts are read: %+v (%v)", read.Acts, err) + } }) // Design 47 H2 §807, H-A-432 (2): an act whose pull request closed unmerged ends not-needed, kept in the record's acts as // ended like performed and could not be performed, and no longer owed. @@ -133,6 +165,10 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. t.Fatalf("said %+v", f.said) } for _, o := range f.said { + // H-A-455 (2): the view is the record without its acts, the end's too, which follows the outcome's acts. + if o.Act == seat.OwedActView && strings.Contains(o.Text, `"acts"`) { + t.Fatalf("the view shows the record's acts: %s", o.Text) + } if o.Act == seat.OwedActComment && (!strings.Contains(o.Text, "tests red") || !strings.Contains(o.Text, "never runs")) { t.Fatalf("comment %q", o.Text) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 1d6d5551f..8424195e9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "fmt" + "strings" "time" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -55,8 +57,11 @@ func (h *Holder) OwedEnded(body []byte) error { // RecordAct is one act the gate owes the forge for a pull request a record holds: said, with when (At, while Outcome is empty) // and as the seat's event says it (Owed, which saying it again at the gate's start repeats), then ended with the forge -// bridge's outcome and when (A-35, design 47 §165). +// bridge's outcome and when (A-35, design 47 §165). Each is an entry of its own, `act/`, its record's +// key the index's field, removed when performed (design 47 H2 §809, H-A-455 (1)). type RecordAct struct { + // Record is its record's key, `.`, the index acts's field. + Record string `json:"record,omitempty"` ID string `json:"id"` Act string `json:"act"` Repository string `json:"repository"` @@ -68,22 +73,87 @@ type RecordAct struct { Owed seat.Owed `json:"owed"` } -// Act writes one act into the record: said (no outcome), or ended; nothing when the record holds it so already. +// TypeAct is the ledger type of an act's entry, and ActsIndex the ledger's index of the acts' entries on their record's key +// (manifest `requires` records' types and indexes; H-A-455 (1)). +const ( + TypeAct = "act" + ActsIndex = "acts" +) + +// ActOperations are an act's operations, in the order the manifest's type act allows them. +var ActOperations = []string{OpAct} + +// ActInstance is the entry of a record's act: `act/`, escaped as a name. +func ActInstance(milestone, id string) (string, error) { + if strings.TrimSpace(id) == "" { + return "", fmt.Errorf("an act of the record %s names its id", milestone) + } + inst := TypeAct + "/" + milestoneToken(milestone+".act."+id) + if !segmentRe.MatchString(strings.TrimPrefix(inst, TypeAct+"/")) { + return "", fmt.Errorf("the act %s of %s cannot be named: %q", id, milestone, inst) + } + return inst, nil +} + +// Act writes one act's entry: said (no outcome), or ended; removed when the forge bridge performed it; nothing when the +// record holds it so already. Its write is measured as every entry's is (H-A-455 (3)). func (rs *Records) Act(r *Record, a RecordAct) error { if had, ok := r.Acts[a.ID]; ok && had.Outcome == a.Outcome { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpAct, a) + inst, err := ActInstance(r.Milestone, a.ID) if err != nil { return err } + if a.Outcome == forgemsg.OwedPerformed { + if err := rs.Ledger.Remove(inst); err != nil { + return err + } + delete(r.Acts, a.ID) + rs.fitted(inst) + return nil + } + a.Record = r.Milestone + if _, err := rs.write(r, inst, OpAct, a, a); err != nil { + return err + } if r.Acts == nil { r.Acts = map[string]RecordAct{} } - r.Acts[a.ID], r.Sequence = a, seq + r.Acts[a.ID] = a return nil } +// acts reads the acts' entries of a record through the index, one query, and each entry; nil when it has none. +func (rs *Records) acts(milestone string) (map[string]RecordAct, error) { + insts, err := rs.Ledger.Query(ActsIndex, moversKey(milestone)) + if err != nil { + return nil, fmt.Errorf("the acts of the record %s cannot be read: %w", milestone, err) + } + var out map[string]RecordAct + for _, inst := range insts { + raw, _, err := rs.Ledger.Get(inst) + if err != nil { + return nil, err + } + if raw == nil { + continue + } + var a RecordAct + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the act %s is not readable: %w", inst, err) + } + if a.Record != milestone || a.ID == "" { + continue + } + if out == nil { + out = map[string]RecordAct{} + } + out[a.ID] = a + } + return out, nil +} + // owedBound is how long an act said may wait for its owed.ended before it is stalled: many times a forge call's time, and // short beside the check's hour (design 47 *What the gate owes the forge*). const owedBound = 15 * time.Minute diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 26ed54871..e80ff1ea7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -32,7 +32,7 @@ const RecordsLedger = "records" // RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate // changes how it writes them. -const RecordForm = 2 +const RecordForm = 3 // The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, // since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 @@ -53,13 +53,13 @@ const ( OpAway = "away" // set-entry into away, by node OpBack = "back" // remove from away, by node OpEnd = "end" // set-fields: ended - OpAct = "act" // set-entry into acts, by id: an act owed the forge with its outcome + OpAct = "act" // an act's entry (TypeAct): set-fields, an act owed the forge with its outcome OpHold = "hold" // a month's list: set-entry into records, by instance OpDrop = "drop" // a month's list: remove from records, by instance ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpAct, OpEnd} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -95,7 +95,8 @@ type Record struct { Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` // Away are the nodes away, by node, each with since when. Away map[string]NodeAway `json:"away,omitempty"` - // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go). + // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go): each an + // entry of its own, joined when read, never on the head entry (design 47 H2 §809, H-A-455 (1)). Acts map[string]RecordAct `json:"acts,omitempty"` Ended *RecordEnded `json:"ended,omitempty"` // Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23). @@ -380,6 +381,8 @@ type Records struct { Class func(fingerprint string) (string, error) // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. TooLarge func(instance string, bytes int, raised bool) + // refused are the entries a write was refused for over EntryLimit, each with its record's key, until the record ends. + refused map[string]string // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -494,6 +497,11 @@ func (rs *Records) Read(instance string) (*Record, error) { return nil, err } r.join(ms) + // A head written before the acts had entries of their own is read for its fields; the acts embedded in it are ignored + // (design 47 H2 §809, H-A-457): the record's acts are its act entries alone. + if r.Acts, err = rs.acts(r.Milestone); err != nil { + return nil, err + } return &r, nil } @@ -549,7 +557,7 @@ func (rs *Records) Open(o RecordOpening) (r *Record, opened bool, err error) { } body := map[string]any{"milestone": milestone, "branch": o.Branch, "source": o.Source, "call": o.Call, "heads": o.Heads, "opened": now, "form": RecordForm, "merge-called": called} - if _, err := rs.Ledger.Apply(inst, OpOpen, body); err != nil { + if _, err := rs.write(&Record{Milestone: milestone}, inst, OpOpen, body, body); err != nil { return nil, false, err } if _, err := rs.Ledger.Apply(heldInstance(now), OpHold, heldEntry{Instance: inst, Opened: now}); err != nil { @@ -677,6 +685,15 @@ func (rs *Records) Sweep(now time.Time) error { return err } } + for id := range r.Acts { + inst, err := ActInstance(r.Milestone, id) + if err != nil { + return err + } + if err := rs.Ledger.Remove(inst); err != nil { + return err + } + } if err := rs.Ledger.Remove(e.Instance); err != nil { return err } @@ -697,15 +714,11 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { } next := *r next.Group = &g - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) + seq, err := rs.write(r, r.Instance, OpGroup, map[string]any{"group": g}, next.head()) if err != nil { return err } r.Group, r.Sequence = &g, seq - rs.fitted(r.Instance) return nil } @@ -716,15 +729,11 @@ func (rs *Records) SetTrace(r *Record, t RecordTrace) error { } next := *r next.Trace = &t - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t}) + seq, err := rs.write(r, r.Instance, OpTrace, map[string]any{"trace": t}, next.head()) if err != nil { return err } r.Trace, r.Sequence = &t, seq - rs.fitted(r.Instance) return nil } @@ -744,15 +753,11 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) } next := *m next.Outcomes = withEntry(m.Outcomes, o.Key, o) - if err := rs.fits(m.Instance, next); err != nil { - return false, err - } - seq, err := rs.Ledger.Apply(m.Instance, OpOutcome, o) + seq, err := rs.write(r, m.Instance, OpOutcome, o, next) if err != nil { return false, err } m.Outcomes, m.Sequence = next.Outcomes, seq - rs.fitted(m.Instance) if r.Outcomes == nil { r.Outcomes = map[string]RecordOutcome{} } @@ -775,15 +780,11 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error { } next := *m next.Entries = withEntry(m.Entries, e.Key, e) - if err := rs.fits(m.Instance, next); err != nil { - return err - } - seq, err := rs.Ledger.Apply(m.Instance, OpEntry, e) + seq, err := rs.write(r, m.Instance, OpEntry, e, next) if err != nil { return err } m.Entries, m.Sequence = next.Entries, seq - rs.fitted(m.Instance) if r.Entries == nil { r.Entries = map[string]RecordEntry{} } @@ -801,9 +802,9 @@ func withEntry[V any](m map[string]V, k string, v V) map[string]V { return out } -// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's. +// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's and no act. func (r Record) head() Record { - r.Entries, r.Outcomes, r.movers = nil, nil, nil + r.Entries, r.Outcomes, r.movers, r.Acts = nil, nil, nil, nil return r } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index e0262dc92..e31a8b5d0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -248,6 +248,11 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T Outcome: OutcomeFailed, Why: why, At: at}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 1 { t.Fatalf("an entry over 64 KiB was taken (%v), findings %v", err, found) } + // H-A-455 (3): an act's write is measured as a mover's is, and the record's end clears its finding too. + if err := rs.Act(read, RecordAct{ID: "view@1.a", Act: "view", Repository: "novox/app", Number: 3, Head: heads["novox/app@3"], + Owed: meshdelivery.Owed{Act: meshdelivery.OwedActView, Text: why}}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 2 { + t.Fatalf("an act over 64 KiB was taken (%v), findings %v", err, found) + } if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 { t.Fatalf("the record's end left %v (%v)", found, err) } @@ -457,7 +462,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T // W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { l := newLedgerFake(t) - for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations} { + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations, + TypeAct: ActOperations} { var got []string for _, op := range l.types[name].Operations { got = append(got, op.Name) @@ -473,6 +479,9 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { if ix := l.types[TypeMover].Indexes; len(ix) != 1 || ix[0].Name != MoversIndex || !slices.Equal(ix[0].On, []string{"record"}) { t.Errorf("the type mover's indexes are %+v, not one %s on its record's key", ix, MoversIndex) } + if ix := l.types[TypeAct].Indexes; len(ix) != 1 || ix[0].Name != ActsIndex || !slices.Equal(ix[0].On, []string{"record"}) { + t.Errorf("the type act's indexes are %+v, not one %s on its record's key (H-A-455 (1))", ix, ActsIndex) + } } // W11.2 (design 47 *The delivery record*): the record's outcomes arrive as events through the gate's consumer: each diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go index 86e785113..5d5bc3640 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -38,7 +38,9 @@ func (rs *Records) NodeAway(r *Record, node string, since time.Time) error { } if _, ok := r.Away[node]; !ok { a := NodeAway{Node: node, Since: since.UTC()} - seq, err := rs.Ledger.Apply(r.Instance, OpAway, a) + next := *r + next.Away = withEntry(r.Away, node, a) + seq, err := rs.write(r, r.Instance, OpAway, a, next.head()) if err != nil { return err } @@ -56,7 +58,14 @@ func (rs *Records) NodeBack(r *Record, node string) error { if _, ok := r.Away[node]; !ok { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpBack, map[string]any{"node": node}) + next := *r + next.Away = map[string]NodeAway{} + for n, a := range r.Away { + if n != node { + next.Away[n] = a + } + } + seq, err := rs.write(r, r.Instance, OpBack, map[string]any{"node": node}, next.head()) if err != nil { return err } @@ -141,7 +150,12 @@ func (rs *Records) end(r *Record, why, by string) error { body["delivery-time"], body["class"] = took.String(), class } } - seq, err := rs.Ledger.Apply(r.Instance, OpEnd, body) + next := *r + next.Ended = &e + if took > 0 { + next.DeliveryTime, next.Class = took.String(), class + } + seq, err := rs.write(r, r.Instance, OpEnd, body, next.head()) if err != nil { return err } @@ -149,11 +163,12 @@ func (rs *Records) end(r *Record, why, by string) error { if took > 0 { r.DeliveryTime, r.Class = took.String(), class } - rs.clearTooLarge(r) rs.keep(r) if rs.Changed != nil { rs.Changed(r, RecordChange{What: "ended", Why: why, Ended: true}) } + // The end clears record-too-large of every entry of the record, the acts its end owed among them (§809, H-A-455 (3)). + rs.clearTooLarge(r) return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go index 3259ce5a1..a6a931398 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go @@ -31,7 +31,9 @@ func (rs *Records) Recovery(repository, commit, why, by string) (meshdelivery.Re if r.Recovery != nil && *r.Recovery == want { return meshdelivery.RecoveryAnswer{By: by, Commit: head, Recovery: true, Repository: repository}, nil } - seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"recovery": want}) + next := *r + next.Recovery = &want + seq, err := rs.write(r, r.Instance, OpTrace, map[string]any{"recovery": want}, next.head()) if err != nil { return meshdelivery.RecoveryAnswer{}, err } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go index da2bde271..da3762abf 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go @@ -57,15 +57,11 @@ func (rs *Records) SetPhases(r *Record, p *runner.PipelineEndedPhases) error { } next := *r next.Phases = p - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpPhases, map[string]any{"phases": p}) + seq, err := rs.write(r, r.Instance, OpPhases, map[string]any{"phases": p}, next.head()) if err != nil { return err } r.Phases, r.Sequence = p, seq - rs.fitted(r.Instance) return nil } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 37f784c26..a6cb5797e 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -103,30 +103,9 @@ "ended": { "type": "object" }, - "id": { - "type": "string" - }, - "act": { - "type": "string" - }, - "repository": { - "type": "string" - }, - "number": { - "type": "integer" - }, - "head": { - "type": "string" - }, - "words": { - "type": "string" - }, "recovery": { "type": "object" }, - "owed": { - "type": "object" - }, "movers": { "type": "integer" }, @@ -208,24 +187,6 @@ "into": "away", "key": "node" }, - { - "name": "act", - "effect": "set-entry", - "fields": [ - "id", - "act", - "repository", - "number", - "head", - "outcome", - "words", - "at", - "owed" - ], - "into": "acts", - "key": "id", - "sequence": true - }, { "name": "end", "effect": "set-fields", @@ -335,6 +296,68 @@ } ] }, + { + "name": "act", + "schema": { + "record": { + "type": "string" + }, + "id": { + "type": "string" + }, + "act": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "head": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "words": { + "type": "string" + }, + "at": { + "type": "string" + }, + "owed": { + "type": "object" + } + }, + "operations": [ + { + "name": "act", + "effect": "set-fields", + "fields": [ + "record", + "id", + "act", + "repository", + "number", + "head", + "outcome", + "words", + "at", + "owed" + ], + "sequence": true + } + ], + "indexes": [ + { + "name": "acts", + "on": [ + "record" + ] + } + ] + }, { "name": "held", "schema": {