diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index a6af04418..e525e2424 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -395,7 +395,7 @@ func (r *runtime) answer(m rpcMessage) { return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) } switch m.Method { - case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery: + case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerQuery, buscalls.LedgerRemove: // The gate's ledger `records` (W11.1), as the tool runner keeps it: the record test's ledger in memory. r.b.mu.Lock() defer r.b.mu.Unlock() @@ -414,6 +414,14 @@ func (r *runtime) answer(m rpcMessage) { } return a, err } + if m.Method == buscalls.LedgerRemove { + // An act performed is removed (design 47 H2 §809, H-A-455 (1)), as the tool runner's ledger removal does. + var in buscalls.LedgerRemoveRequest + if err := json.Unmarshal(m.Params, &in); err != nil { + return nil, err + } + return buscalls.LedgerRemoved{}, r.b.ledger.Remove(in.Instance) + } if m.Method == buscalls.LedgerGet { var in buscalls.InstanceRequest if err := json.Unmarshal(m.Params, &in); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go index 62f9e33bb..c21674dd2 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mergecall_bus_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "testing" "time" @@ -110,6 +111,75 @@ func TestAGateWholeTakesAMergeCallAndWatchesTheHeadsOverARealBus(t *testing.T) { want.Round(time.Minute)) } + // GT1 (design 47 H2 §809; H-A-385, H-A-391, H-A-453 (3), H-A-455): a record of 126 movers writes 127 entries, its head + // and one per mover, and its acts owed the forge each an entry of its own, none over 64 KiB, the head counting 126 and + // holding no act; an entry made too large raises record-too-large, and the record's end clears it. + call2, merge2 := "00000000000000c2", strings.Repeat("e", 40) + instance2, _ := RecordInstance(recordMilestone(branch, 2)) + b.publish(t, "mesh.seat.git.event.merge-called."+call2, map[string]any{"call": call2, "by": "lead@shanks", "account": "acct-s", + "at": time.Now().UTC().Format(time.RFC3339), "members": []any{map[string]any{"repository": "novox/app", "number": 6, + "branch": branch, "head": strings.Repeat("f", 40), "commit": merge2}}}) + rt.until("the record of the second merge call", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.ledger.states[instance2] != nil + }) + var taken []any + for i := range 126 { + module := "mesh-module-" + strings.Repeat("x", 3) + fmt.Sprintf("%03d", i) + taken = append(taken, map[string]any{"module": module, "commit": merge2, "fingerprint": fmt.Sprintf("%064x", i), "repository": "novox/app"}) + } + b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 2, "revision": 2, + "call": call2, "entries": taken}) + rt.until("the record's 126 movers taken in", eventWithin, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + last, _ := MoverInstance(recordMilestone(branch, 2), taken[125].(map[string]any)["module"].(string)) + return strings.Contains(string(b.ledger.states[instance2]), `"movers":126`) && + strings.Contains(string(b.ledger.states[last]), `"outcome":"`+OutcomeTakenIn+`"`) + }) + b.mu.Lock() + entries, acts, over := 0, 0, []string{} + for inst, raw := range b.ledger.states { + if string(raw) == "null" { + continue + } + var of struct { + Milestone string `json:"milestone"` + Record string `json:"record"` + } + _ = json.Unmarshal(raw, &of) + if of.Milestone != recordMilestone(branch, 2) && of.Record != recordMilestone(branch, 2) { + continue + } + if strings.HasPrefix(inst, TypeAct+"/") { + acts++ + } else { + entries++ + } + if len(raw) > EntryLimit { + over = append(over, fmt.Sprintf("%s holds %d bytes", inst, len(raw))) + } + } + headRaw := string(b.ledger.states[instance2]) + b.mu.Unlock() + if entries != 127 || acts == 0 || len(over) > 0 || strings.Contains(headRaw, `"acts"`) { + t.Fatalf("the record of 126 movers writes %d entries and %d acts, over 64 KiB: %v; its head: %.300s", entries, acts, over, headRaw) + } + if findingHeld(ctx, b, RecordTooLarge, "") { + t.Fatal("record-too-large is raised for a record whose entries are within 64 KiB") + } + // The first mover's take-in said again with a why past 64 KiB: its entry's write is refused and raises record-too-large; + // the record's end clears it. + first := taken[0].(map[string]any) + tooLarge, _ := MoverInstance(recordMilestone(branch, 2), first["module"].(string)) + b.publish(t, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", map[string]any{"branch": branch, "source": 2, "revision": 2, + "call": call2, "why": strings.Repeat("w", EntryLimit), "entries": []any{first}}) + rt.until("record-too-large raised for the mover's entry", eventWithin, func() bool { return findingHeld(ctx, b, RecordTooLarge, tooLarge) }) + b.publish(t, "mesh.seat.mesh-wants.event.pipeline-wanted."+escaped+".2", map[string]any{"v": 1, "branch": branch, + "source": 2, "pipeline": recordMilestone(branch, 2), "outcome": "running"}) + rt.until("record-too-large cleared at the record's end", eventWithin, func() bool { return !findingHeld(ctx, b, RecordTooLarge, "") }) + // The seed (§768): every watched branch's first source revision at its head; its head-moved then holds nothing. seeded := strings.Repeat("5", 40) b.mu.Lock() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go index 4fc43b265..f70f710ed 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/movers.go @@ -1,12 +1,13 @@ package main // A record's entries, one per module (design 47 H2 §809; H-A-385, H-A-391): the head entry `record/` keeps -// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module; each mover, the -// catalog half's too, has its own entry `mover/` with its pipeline, version and the module's step -// outcomes per node (its machines and their running times among them). No entry names two modules. A record is joined -// from its head and the movers the ledger's index `movers` names under the record's key, one query. The gate refuses an -// entry over 64 KiB with the finding record-too-large, cleared when that entry is next written within 64 KiB or at the -// record's end. +// every record-wide field, its outcome, the count of its movers and §810's times, nothing per module and no act; each +// mover, the catalog half's too, has its own entry `mover/` with its pipeline, version and the +// module's step outcomes per node (its machines and their running times among them); each act owed the forge has its own +// entry `act/`, removed when performed (H-A-455 (1)). No entry names two modules. A record is joined +// from its head and the movers and acts the ledger's indexes `movers` and `acts` name under the record's key, one query +// each. The gate refuses a write over 64 KiB to any entry, the head's, a mover's or an act's, with the finding +// record-too-large, cleared when that entry is next written within 64 KiB or at the record's end (H-A-455 (3)). import ( "encoding/json" @@ -85,7 +86,7 @@ func MoverInstance(milestone, module string) (string, error) { return inst, nil } -// moversKey is the index key that names every mover of a record: its key's token, then the rest. +// moversKey is the index key that names every mover or act of a record: its key's token, then the rest. func moversKey(milestone string) string { return strings.TrimSuffix(buscalls.IndexKey([]string{milestone}, "x"), ".x") + ".>" } @@ -160,7 +161,7 @@ func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) class = firstOf(c, catalogue.ClassLeaf) } m := &Mover{Instance: inst, Record: r.Milestone, Module: module, Class: class} - seq, err := rs.Ledger.Apply(inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}) + seq, err := rs.write(r, inst, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": class}, m) if err != nil { return nil, err } @@ -169,7 +170,9 @@ func (rs *Records) mover(r *Record, module, fingerprint string) (*Mover, error) r.movers = map[string]*Mover{} } r.movers[module] = m - hseq, err := rs.Ledger.Apply(r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}) + next := *r + next.MoverCount = len(r.movers) + hseq, err := rs.write(r, r.Instance, OpMovers, map[string]any{"movers": len(r.movers)}, next.head()) if err != nil { return m, fmt.Errorf("the mover %s is written and the record %s's count of its movers is not: %w", module, r.Instance, err) } @@ -188,21 +191,33 @@ func (rs *Records) SetMover(r *Record, module, fingerprint, pipeline, version st if next.Pipeline == m.Pipeline && next.Version == m.Version { return nil } - if err := rs.fits(m.Instance, next); err != nil { - return err - } - seq, err := rs.Ledger.Apply(m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, - "pipeline": next.Pipeline, "version": next.Version}) + seq, err := rs.write(r, m.Instance, OpMover, map[string]any{"record": r.Milestone, "module": module, "class": m.Class, + "pipeline": next.Pipeline, "version": next.Version}, next) if err != nil { return err } m.Pipeline, m.Version, m.Sequence = next.Pipeline, next.Version, seq - rs.fitted(m.Instance) return nil } -// fits refuses an entry whose state would pass EntryLimit, raising record-too-large for it (§809). -func (rs *Records) fits(instance string, state any) error { +// write is one write to an entry of a record, measured first: the entry's state after it (the head's without its movers +// and acts) is refused over EntryLimit with record-too-large, and a write taken clears the entry's finding (§809, H-A-455 +// (3)). +func (rs *Records) write(r *Record, instance, operation string, body, state any) (uint64, error) { + if err := rs.fits(r, instance, state); err != nil { + return 0, err + } + seq, err := rs.Ledger.Apply(instance, operation, body) + if err != nil { + return 0, err + } + rs.fitted(instance) + return seq, nil +} + +// fits refuses an entry of r whose state would pass EntryLimit, raising record-too-large for it (§809); the entry is +// remembered with its record, so the record's end clears it whichever entry it was (H-A-455 (3)). +func (rs *Records) fits(r *Record, instance string, state any) error { raw, err := json.Marshal(state) if err != nil { return err @@ -210,6 +225,12 @@ func (rs *Records) fits(instance string, state any) error { if len(raw) <= EntryLimit { return nil } + rs.mu.Lock() + if rs.refused == nil { + rs.refused = map[string]string{} + } + rs.refused[instance] = r.Milestone + rs.mu.Unlock() if rs.TooLarge != nil { rs.TooLarge(instance, len(raw), true) } @@ -223,10 +244,27 @@ func (rs *Records) fitted(instance string) { } } -// clearTooLarge clears record-too-large for every entry of a record at its end. +// clearTooLarge clears record-too-large for every entry of a record at its end: its head, its movers, its acts and every +// entry of it a write was refused for. func (rs *Records) clearTooLarge(r *Record) { - rs.fitted(r.Instance) + insts := []string{r.Instance} for _, m := range r.movers { - rs.fitted(m.Instance) + insts = append(insts, m.Instance) + } + for id := range r.Acts { + if inst, err := ActInstance(r.Milestone, id); err == nil { + insts = append(insts, inst) + } + } + rs.mu.Lock() + for inst, of := range rs.refused { + if of == r.Milestone { + insts = append(insts, inst) + delete(rs.refused, inst) + } + } + rs.mu.Unlock() + for _, inst := range insts { + rs.fitted(inst) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go index e27918d7d..b297ff9dc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts.go @@ -12,9 +12,9 @@ import ( // What the gate owes the forge for a change of a delivery record (design 47 G A-39, A-35, §158, §161 to §167): every change of // a record, a step's outcome applied or the record ended, says a line on the commit's note of each member pull request's head -// (` record : ()`) and replaces each member's delivery view with what `show` answers for the -// record; a comment is said only when the record ends failed or held, naming why. Each act goes through owe (owed.go): kept in -// the record's acts, said as the seat's event owed, and ended on owed.ended. +// (` record : ()`) and replaces each member's delivery view with the record's summary, its head's +// fields and a count of its movers per outcome (H-A-459); a comment is said only when the record ends failed or held, naming why. Each act goes through owe (owed.go): kept as +// an entry of its own among the record's acts, said as the seat's event owed, and ended on owed.ended. // RecordChange is one change of a record: what changed and why; Ended when it is the record's end. type RecordChange struct { @@ -39,7 +39,7 @@ func (h *Holder) recordChanged(r *Record, c RecordChange) { } now := h.Now().UTC() line := fmt.Sprintf("%s record %s: %s (%s)", now.Format("2006-01-02T15:04:05Z"), r.Instance, c.What, c.Why) - view, err := json.MarshalIndent(map[string]any{"records": []recordView{viewOf(r)}}, "", " ") + view, err := json.MarshalIndent(map[string]any{"records": []recordSummary{summaryOf(r)}}, "", " ") if err != nil { h.Refuse(fmt.Errorf("the record %s cannot be shown on its pull requests: %w", r.Instance, err)) return @@ -70,6 +70,36 @@ func (h *Holder) recordChanged(r *Record, c RecordChange) { } } +// recordSummary is the record a view act shows (design 47 H2 §809, H-A-455 (2), H-A-459): the head's fields, without its +// acts, and a count of its movers per outcome, never the movers' entries, so a view act of any record fits 64 KiB; the +// whole record stays readable through gate_record. +type recordSummary struct { + recordView + MoversPerOutcome map[string]int `json:"movers-per-outcome,omitempty"` +} + +// summaryOf is the record a view act shows. +func summaryOf(r *Record) recordSummary { + v := viewOf(r) + head := r.head() + v.Record = &head + per := map[string]map[string]bool{} + for _, o := range r.Outcomes { + if per[o.Outcome] == nil { + per[o.Outcome] = map[string]bool{} + } + per[o.Outcome][o.Module] = true + } + out := recordSummary{recordView: v} + for outcome, movers := range per { + if out.MoversPerOutcome == nil { + out.MoversPerOutcome = map[string]int{} + } + out.MoversPerOutcome[outcome] = len(movers) + } + return out +} + // endedFailedOrHeld is the comment a record ended says when one of its outcomes is failed or held, naming why; "" otherwise. func endedFailedOrHeld(r *Record) string { var lines []string diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go index f7331bc9e..30c7c83b0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedacts_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "strings" "testing" "time" @@ -41,7 +42,7 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. return } t.Run("an outcome applied says the note and the view, no comment", func(t *testing.T) { - _, rs, r, f := setup(t) + h, rs, r, f := setup(t) if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil { t.Fatal(err) } @@ -62,7 +63,8 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if o.Act == seat.OwedActNote && (o.Line != "2026-11-11T21:00:00Z record "+r.Instance+": mesh-x on ace build built (built it)" || o.Commit != head) { t.Fatalf("note %+v", o) } - if o.Act == seat.OwedActView && !strings.Contains(o.Text, r.Instance) { + // H-A-459: the view is the record's head fields and a count of movers per outcome, never the movers' entries. + if o.Act == seat.OwedActView && (!strings.Contains(o.Text, r.Instance) || !strings.Contains(strings.Join(strings.Fields(o.Text), ""), `"movers-per-outcome":{"built":1}`) || strings.Contains(o.Text, `"outcomes"`) || strings.Contains(o.Text, `"entries"`)) { t.Fatalf("view %q", o.Text) } } @@ -70,6 +72,36 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. if _, err := rs.Outcome(r, RecordOutcome{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace", Step: "build", Outcome: OutcomeBuilt, Why: "built it"}); err != nil || len(f.said) != 2 { t.Fatalf("said again: %v %d", err, len(f.said)) } + // Design 47 H2 §809, H-A-455 (1): each act is an entry of its own, act/, naming its record; the + // head entry holds none, and an act performed is removed. + l := rs.Ledger.(*ledgerFake) + for _, o := range f.said { + inst, err := ActInstance(r.Milestone, o.ID) + if err != nil || !strings.Contains(string(l.states[inst]), `"record":"main.47"`) { + t.Fatalf("the act %s has no entry of its own at %s: %s (%v)", o.ID, inst, l.states[inst], err) + } + } + if strings.Contains(string(l.states[r.Instance]), `"acts"`) { + t.Fatalf("the head entry holds acts: %s", l.states[r.Instance]) + } + note := f.said[0] + body, err := forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: note.ID, Outcome: forgemsg.OwedPerformed, At: at}) + if err != nil { + t.Fatal(err) + } + if err := h.OwedEnded(body); err != nil { + t.Fatal(err) + } + inst, _ := ActInstance(r.Milestone, note.ID) + if string(l.states[inst]) != "null" || h.Status("")["owed"] != 1 { + t.Fatalf("the act performed is kept as %s, %v owed", l.states[inst], h.Status("")["owed"]) + } + // H-A-457: a head written before, its acts embedded, is read for its fields; those acts are ignored. + l.states[r.Instance] = json.RawMessage(strings.Replace(string(l.states[r.Instance]), `{`, `{"acts":{"old":{"id":"old","act":"view"}},`, 1)) + read, err := rs.Read(r.Instance) + if _, old := read.Acts["old"]; err != nil || old || len(read.Acts) != 1 || read.Milestone != "main.47" { + t.Fatalf("an old head's embedded acts are read: %+v (%v)", read.Acts, err) + } }) // Design 47 H2 §807, H-A-432 (2): an act whose pull request closed unmerged ends not-needed, kept in the record's acts as // ended like performed and could not be performed, and no longer owed. @@ -133,6 +165,10 @@ func TestARecordsChangesOweTheNoteTheViewAndWhenFailedOrHeldAComment(t *testing. t.Fatalf("said %+v", f.said) } for _, o := range f.said { + // H-A-455 (2): the view is the record without its acts, the end's too, which follows the outcome's acts. + if o.Act == seat.OwedActView && strings.Contains(o.Text, `"acts"`) { + t.Fatalf("the view shows the record's acts: %s", o.Text) + } if o.Act == seat.OwedActComment && (!strings.Contains(o.Text, "tests red") || !strings.Contains(o.Text, "never runs")) { t.Fatalf("comment %q", o.Text) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 1d6d5551f..8424195e9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "fmt" + "strings" "time" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -55,8 +57,11 @@ func (h *Holder) OwedEnded(body []byte) error { // RecordAct is one act the gate owes the forge for a pull request a record holds: said, with when (At, while Outcome is empty) // and as the seat's event says it (Owed, which saying it again at the gate's start repeats), then ended with the forge -// bridge's outcome and when (A-35, design 47 §165). +// bridge's outcome and when (A-35, design 47 §165). Each is an entry of its own, `act/`, its record's +// key the index's field, removed when performed (design 47 H2 §809, H-A-455 (1)). type RecordAct struct { + // Record is its record's key, `.`, the index acts's field. + Record string `json:"record,omitempty"` ID string `json:"id"` Act string `json:"act"` Repository string `json:"repository"` @@ -68,22 +73,87 @@ type RecordAct struct { Owed seat.Owed `json:"owed"` } -// Act writes one act into the record: said (no outcome), or ended; nothing when the record holds it so already. +// TypeAct is the ledger type of an act's entry, and ActsIndex the ledger's index of the acts' entries on their record's key +// (manifest `requires` records' types and indexes; H-A-455 (1)). +const ( + TypeAct = "act" + ActsIndex = "acts" +) + +// ActOperations are an act's operations, in the order the manifest's type act allows them. +var ActOperations = []string{OpAct} + +// ActInstance is the entry of a record's act: `act/`, escaped as a name. +func ActInstance(milestone, id string) (string, error) { + if strings.TrimSpace(id) == "" { + return "", fmt.Errorf("an act of the record %s names its id", milestone) + } + inst := TypeAct + "/" + milestoneToken(milestone+".act."+id) + if !segmentRe.MatchString(strings.TrimPrefix(inst, TypeAct+"/")) { + return "", fmt.Errorf("the act %s of %s cannot be named: %q", id, milestone, inst) + } + return inst, nil +} + +// Act writes one act's entry: said (no outcome), or ended; removed when the forge bridge performed it; nothing when the +// record holds it so already. Its write is measured as every entry's is (H-A-455 (3)). func (rs *Records) Act(r *Record, a RecordAct) error { if had, ok := r.Acts[a.ID]; ok && had.Outcome == a.Outcome { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpAct, a) + inst, err := ActInstance(r.Milestone, a.ID) if err != nil { return err } + if a.Outcome == forgemsg.OwedPerformed { + if err := rs.Ledger.Remove(inst); err != nil { + return err + } + delete(r.Acts, a.ID) + rs.fitted(inst) + return nil + } + a.Record = r.Milestone + if _, err := rs.write(r, inst, OpAct, a, a); err != nil { + return err + } if r.Acts == nil { r.Acts = map[string]RecordAct{} } - r.Acts[a.ID], r.Sequence = a, seq + r.Acts[a.ID] = a return nil } +// acts reads the acts' entries of a record through the index, one query, and each entry; nil when it has none. +func (rs *Records) acts(milestone string) (map[string]RecordAct, error) { + insts, err := rs.Ledger.Query(ActsIndex, moversKey(milestone)) + if err != nil { + return nil, fmt.Errorf("the acts of the record %s cannot be read: %w", milestone, err) + } + var out map[string]RecordAct + for _, inst := range insts { + raw, _, err := rs.Ledger.Get(inst) + if err != nil { + return nil, err + } + if raw == nil { + continue + } + var a RecordAct + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the act %s is not readable: %w", inst, err) + } + if a.Record != milestone || a.ID == "" { + continue + } + if out == nil { + out = map[string]RecordAct{} + } + out[a.ID] = a + } + return out, nil +} + // owedBound is how long an act said may wait for its owed.ended before it is stalled: many times a forge call's time, and // short beside the check's hour (design 47 *What the gate owes the forge*). const owedBound = 15 * time.Minute diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 26ed54871..e80ff1ea7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -32,7 +32,7 @@ const RecordsLedger = "records" // RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate // changes how it writes them. -const RecordForm = 2 +const RecordForm = 3 // The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, // since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 @@ -53,13 +53,13 @@ const ( OpAway = "away" // set-entry into away, by node OpBack = "back" // remove from away, by node OpEnd = "end" // set-fields: ended - OpAct = "act" // set-entry into acts, by id: an act owed the forge with its outcome + OpAct = "act" // an act's entry (TypeAct): set-fields, an act owed the forge with its outcome OpHold = "hold" // a month's list: set-entry into records, by instance OpDrop = "drop" // a month's list: remove from records, by instance ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpAct, OpEnd} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpMovers, OpPhases, OpAway, OpBack, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -95,7 +95,8 @@ type Record struct { Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` // Away are the nodes away, by node, each with since when. Away map[string]NodeAway `json:"away,omitempty"` - // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go). + // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go): each an + // entry of its own, joined when read, never on the head entry (design 47 H2 §809, H-A-455 (1)). Acts map[string]RecordAct `json:"acts,omitempty"` Ended *RecordEnded `json:"ended,omitempty"` // Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23). @@ -380,6 +381,8 @@ type Records struct { Class func(fingerprint string) (string, error) // TooLarge raises (raised) or clears record-too-large for an entry (§809); nil raises none. TooLarge func(instance string, bytes int, raised bool) + // refused are the entries a write was refused for over EntryLimit, each with its record's key, until the record ends. + refused map[string]string // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -494,6 +497,11 @@ func (rs *Records) Read(instance string) (*Record, error) { return nil, err } r.join(ms) + // A head written before the acts had entries of their own is read for its fields; the acts embedded in it are ignored + // (design 47 H2 §809, H-A-457): the record's acts are its act entries alone. + if r.Acts, err = rs.acts(r.Milestone); err != nil { + return nil, err + } return &r, nil } @@ -549,7 +557,7 @@ func (rs *Records) Open(o RecordOpening) (r *Record, opened bool, err error) { } body := map[string]any{"milestone": milestone, "branch": o.Branch, "source": o.Source, "call": o.Call, "heads": o.Heads, "opened": now, "form": RecordForm, "merge-called": called} - if _, err := rs.Ledger.Apply(inst, OpOpen, body); err != nil { + if _, err := rs.write(&Record{Milestone: milestone}, inst, OpOpen, body, body); err != nil { return nil, false, err } if _, err := rs.Ledger.Apply(heldInstance(now), OpHold, heldEntry{Instance: inst, Opened: now}); err != nil { @@ -677,6 +685,15 @@ func (rs *Records) Sweep(now time.Time) error { return err } } + for id := range r.Acts { + inst, err := ActInstance(r.Milestone, id) + if err != nil { + return err + } + if err := rs.Ledger.Remove(inst); err != nil { + return err + } + } if err := rs.Ledger.Remove(e.Instance); err != nil { return err } @@ -697,15 +714,11 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { } next := *r next.Group = &g - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) + seq, err := rs.write(r, r.Instance, OpGroup, map[string]any{"group": g}, next.head()) if err != nil { return err } r.Group, r.Sequence = &g, seq - rs.fitted(r.Instance) return nil } @@ -716,15 +729,11 @@ func (rs *Records) SetTrace(r *Record, t RecordTrace) error { } next := *r next.Trace = &t - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t}) + seq, err := rs.write(r, r.Instance, OpTrace, map[string]any{"trace": t}, next.head()) if err != nil { return err } r.Trace, r.Sequence = &t, seq - rs.fitted(r.Instance) return nil } @@ -744,15 +753,11 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) } next := *m next.Outcomes = withEntry(m.Outcomes, o.Key, o) - if err := rs.fits(m.Instance, next); err != nil { - return false, err - } - seq, err := rs.Ledger.Apply(m.Instance, OpOutcome, o) + seq, err := rs.write(r, m.Instance, OpOutcome, o, next) if err != nil { return false, err } m.Outcomes, m.Sequence = next.Outcomes, seq - rs.fitted(m.Instance) if r.Outcomes == nil { r.Outcomes = map[string]RecordOutcome{} } @@ -775,15 +780,11 @@ func (rs *Records) AddEntry(r *Record, e RecordEntry) error { } next := *m next.Entries = withEntry(m.Entries, e.Key, e) - if err := rs.fits(m.Instance, next); err != nil { - return err - } - seq, err := rs.Ledger.Apply(m.Instance, OpEntry, e) + seq, err := rs.write(r, m.Instance, OpEntry, e, next) if err != nil { return err } m.Entries, m.Sequence = next.Entries, seq - rs.fitted(m.Instance) if r.Entries == nil { r.Entries = map[string]RecordEntry{} } @@ -801,9 +802,9 @@ func withEntry[V any](m map[string]V, k string, v V) map[string]V { return out } -// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's. +// head is the record's head entry as the ledger holds it: every record-wide field, none of a mover's and no act. func (r Record) head() Record { - r.Entries, r.Outcomes, r.movers = nil, nil, nil + r.Entries, r.Outcomes, r.movers, r.Acts = nil, nil, nil, nil return r } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index e0262dc92..e31a8b5d0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -248,6 +248,11 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T Outcome: OutcomeFailed, Why: why, At: at}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 1 { t.Fatalf("an entry over 64 KiB was taken (%v), findings %v", err, found) } + // H-A-455 (3): an act's write is measured as a mover's is, and the record's end clears its finding too. + if err := rs.Act(read, RecordAct{ID: "view@1.a", Act: "view", Repository: "novox/app", Number: 3, Head: heads["novox/app@3"], + Owed: meshdelivery.Owed{Act: meshdelivery.OwedActView, Text: why}}); err == nil || !strings.Contains(err.Error(), RecordTooLarge) || len(found) != 2 { + t.Fatalf("an act over 64 KiB was taken (%v), findings %v", err, found) + } if err := rs.End(read, "every present node runs its entries", ""); err != nil || len(found) != 0 { t.Fatalf("the record's end left %v (%v)", found, err) } @@ -457,7 +462,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T // W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { l := newLedgerFake(t) - for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations} { + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations, TypeMover: MoverOperations, + TypeAct: ActOperations} { var got []string for _, op := range l.types[name].Operations { got = append(got, op.Name) @@ -473,6 +479,9 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { if ix := l.types[TypeMover].Indexes; len(ix) != 1 || ix[0].Name != MoversIndex || !slices.Equal(ix[0].On, []string{"record"}) { t.Errorf("the type mover's indexes are %+v, not one %s on its record's key", ix, MoversIndex) } + if ix := l.types[TypeAct].Indexes; len(ix) != 1 || ix[0].Name != ActsIndex || !slices.Equal(ix[0].On, []string{"record"}) { + t.Errorf("the type act's indexes are %+v, not one %s on its record's key (H-A-455 (1))", ix, ActsIndex) + } } // W11.2 (design 47 *The delivery record*): the record's outcomes arrive as events through the gate's consumer: each diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go index 86e785113..5d5bc3640 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -38,7 +38,9 @@ func (rs *Records) NodeAway(r *Record, node string, since time.Time) error { } if _, ok := r.Away[node]; !ok { a := NodeAway{Node: node, Since: since.UTC()} - seq, err := rs.Ledger.Apply(r.Instance, OpAway, a) + next := *r + next.Away = withEntry(r.Away, node, a) + seq, err := rs.write(r, r.Instance, OpAway, a, next.head()) if err != nil { return err } @@ -56,7 +58,14 @@ func (rs *Records) NodeBack(r *Record, node string) error { if _, ok := r.Away[node]; !ok { return nil } - seq, err := rs.Ledger.Apply(r.Instance, OpBack, map[string]any{"node": node}) + next := *r + next.Away = map[string]NodeAway{} + for n, a := range r.Away { + if n != node { + next.Away[n] = a + } + } + seq, err := rs.write(r, r.Instance, OpBack, map[string]any{"node": node}, next.head()) if err != nil { return err } @@ -141,7 +150,12 @@ func (rs *Records) end(r *Record, why, by string) error { body["delivery-time"], body["class"] = took.String(), class } } - seq, err := rs.Ledger.Apply(r.Instance, OpEnd, body) + next := *r + next.Ended = &e + if took > 0 { + next.DeliveryTime, next.Class = took.String(), class + } + seq, err := rs.write(r, r.Instance, OpEnd, body, next.head()) if err != nil { return err } @@ -149,11 +163,12 @@ func (rs *Records) end(r *Record, why, by string) error { if took > 0 { r.DeliveryTime, r.Class = took.String(), class } - rs.clearTooLarge(r) rs.keep(r) if rs.Changed != nil { rs.Changed(r, RecordChange{What: "ended", Why: why, Ended: true}) } + // The end clears record-too-large of every entry of the record, the acts its end owed among them (§809, H-A-455 (3)). + rs.clearTooLarge(r) return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go index 3259ce5a1..a6a931398 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go @@ -31,7 +31,9 @@ func (rs *Records) Recovery(repository, commit, why, by string) (meshdelivery.Re if r.Recovery != nil && *r.Recovery == want { return meshdelivery.RecoveryAnswer{By: by, Commit: head, Recovery: true, Repository: repository}, nil } - seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"recovery": want}) + next := *r + next.Recovery = &want + seq, err := rs.write(r, r.Instance, OpTrace, map[string]any{"recovery": want}, next.head()) if err != nil { return meshdelivery.RecoveryAnswer{}, err } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go index da2bde271..da3762abf 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go @@ -57,15 +57,11 @@ func (rs *Records) SetPhases(r *Record, p *runner.PipelineEndedPhases) error { } next := *r next.Phases = p - if err := rs.fits(r.Instance, next.head()); err != nil { - return err - } - seq, err := rs.Ledger.Apply(r.Instance, OpPhases, map[string]any{"phases": p}) + seq, err := rs.write(r, r.Instance, OpPhases, map[string]any{"phases": p}, next.head()) if err != nil { return err } r.Phases, r.Sequence = p, seq - rs.fitted(r.Instance) return nil } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 37f784c26..a6cb5797e 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -103,30 +103,9 @@ "ended": { "type": "object" }, - "id": { - "type": "string" - }, - "act": { - "type": "string" - }, - "repository": { - "type": "string" - }, - "number": { - "type": "integer" - }, - "head": { - "type": "string" - }, - "words": { - "type": "string" - }, "recovery": { "type": "object" }, - "owed": { - "type": "object" - }, "movers": { "type": "integer" }, @@ -208,24 +187,6 @@ "into": "away", "key": "node" }, - { - "name": "act", - "effect": "set-entry", - "fields": [ - "id", - "act", - "repository", - "number", - "head", - "outcome", - "words", - "at", - "owed" - ], - "into": "acts", - "key": "id", - "sequence": true - }, { "name": "end", "effect": "set-fields", @@ -335,6 +296,68 @@ } ] }, + { + "name": "act", + "schema": { + "record": { + "type": "string" + }, + "id": { + "type": "string" + }, + "act": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "head": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "words": { + "type": "string" + }, + "at": { + "type": "string" + }, + "owed": { + "type": "object" + } + }, + "operations": [ + { + "name": "act", + "effect": "set-fields", + "fields": [ + "record", + "id", + "act", + "repository", + "number", + "head", + "outcome", + "words", + "at", + "owed" + ], + "sequence": true + } + ], + "indexes": [ + { + "name": "acts", + "on": [ + "record" + ] + } + ] + }, { "name": "held", "schema": {