diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 546065b58..2391c55ee 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -198,12 +198,24 @@ func tools(h *Holder, l *listening) []stdio.Tool { {Name: seat + "deliveries", Description: "Every delivery record the gate keeps, as it is kept (design 47 G A-24): the open ones, and those that ended in " + "the last thirty days (all: every one kept), each with its milestone or group, its pull requests, its entries' outcomes " + - "per node, its acts owed the forge and why it is not ended. Narrowed by state (open or ended), repository or group.", + "per node, its acts owed the forge and why it is not ended. Narrowed by state (open or ended), repository, group, module, " + + "outcome or since while it reads, at most 768 KB a page with capped, total and the next page's cursor (design 47 H2 §811).", Input: map[string]any{"type": "object", "properties": map[string]any{"state": str("open or ended"), "repository": str("owner/repository"), "group": str("a group's id (its branch name)"), - "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}}, + "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}, + "module": str("only the deliveries that move this module"), "outcome": str("only the deliveries that ended with this outcome"), + "since": str("only the deliveries opened at or after this time, RFC 3339"), "next": str("the cursor the previous page answered")}}, Run: func(a map[string]any) (any, error) { - return h.Deliveries(strArg(a, "state"), strArg(a, "repository"), strArg(a, "group"), boolArg(a, "all")) + f := DeliveriesFilter{State: strArg(a, "state"), Repository: strArg(a, "repository"), Group: strArg(a, "group"), + Module: strArg(a, "module"), Outcome: strArg(a, "outcome"), All: boolArg(a, "all"), Next: strArg(a, "next")} + if since := strArg(a, "since"); since != "" { + t, err := time.Parse(time.RFC3339, since) + if err != nil { + return nil, fmt.Errorf("deliveries: since %q is no RFC 3339 time", since) + } + f.Since = t + } + return h.Deliveries(f) }}, {Name: seat + "show", Description: "One delivery record whole, as it is kept: by its instance, its milestone, its group's id or the id of one of " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 90c0d1751..d238d801d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -371,7 +371,7 @@ func TestTheGroupsFromBranchNamesAreGone(t *testing.T) { if _, _, err := h.Records.Open(opening("main", 7, map[string]string{"novox/app#3": strings.Repeat("a", 40)})); err != nil { t.Fatal(err) } - d, err := h.Deliveries("", "", "", true) + d, err := h.Deliveries(DeliveriesFilter{All: true}) if err != nil || len(d["deliveries"].([]recordView)) != 1 { t.Fatalf("deliveries from the records: %v %v", d, err) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 487886b12..aeb067199 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -569,6 +569,69 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { t.Fatalf("%+v %v", lines, err) } }) + // deliveries filters while it reads and pages at 768 KB with capped, total and a cursor (design 47 H2 §811, §779). + t.Run("deliveries by repository over 4 MB of records answers under 768 KB, page by page", func(t *testing.T) { + rs := &Records{Ledger: newLedgerFake(t), Now: func() time.Time { return at }} + big := make([]string, 400) + for i := range big { + big[i] = strings.Repeat("c", 40) + } + for i := range 300 { + repo := "novox/app" + if i%2 == 1 { + repo = "novox/other" + } + r, _, err := rs.Open(opening("main", int64(i+1), map[string]string{fmt.Sprintf("%s@%d", repo, i): strings.Repeat("1", 40)})) + if err != nil { + t.Fatal(err) + } + if err := rs.SetTrace(r, RecordTrace{Commits: big}); err != nil { + t.Fatal(err) + } + if i == 4 { + if err := rs.AddEntry(r, RecordEntry{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace"}); err != nil { + t.Fatal(err) + } + } + } + h := &Holder{Records: rs, Now: func() time.Time { return at }} + kept, _ := rs.Kept() + raw, _ := json.Marshal(kept) + if whole, _ := h.Deliveries(DeliveriesFilter{All: true}); len(raw) < 4<<20 || !whole["capped"].(bool) || whole["total"].(int) != 300 { + t.Fatalf("the records hold %d bytes in all, total %v", len(raw), whole["total"]) + } + seen, next := map[string]bool{}, "" + for pages := 0; ; pages++ { + page, err := h.Deliveries(DeliveriesFilter{Repository: "novox/app", All: true, Next: next}) + if err != nil { + t.Fatal(err) + } + if raw, _ := json.Marshal(page); len(raw) > 768<<10 || page["total"].(int) != 150 { + t.Fatalf("a page of %d bytes, total %v", len(raw), page["total"]) + } + for _, v := range page["deliveries"].([]recordView) { + if seen[v.Instance] || !holdsRepository(v.Record, "novox/app") { + t.Fatalf("%s answered twice or of another repository", v.Instance) + } + seen[v.Instance] = true + } + if next = page["next"].(string); next == "" || pages > 20 { + break + } + } + if len(seen) != 150 { + t.Errorf("the pages answered %d of 150 records", len(seen)) + } + if m, _ := h.Deliveries(DeliveriesFilter{Module: "mesh-x", All: true}); m["total"].(int) != 1 { + t.Errorf("by module: %v", m["total"]) + } + if s, _ := h.Deliveries(DeliveriesFilter{Since: at.Add(time.Hour), All: true}); s["total"].(int) != 0 { + t.Errorf("since an hour after every opening: %v", s["total"]) + } + if _, err := h.Deliveries(DeliveriesFilter{Next: "not-a-cursor"}); err == nil { + t.Error("a cursor this verb never answered was taken") + } + }) t.Run("gate_record_end ends a record only once the hand-act log took it", func(t *testing.T) { rs, r, said := setup(t) log := &handActsFake{refuse: errors.New("no responders")} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index ecf5170d9..78a9be406 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -1,10 +1,14 @@ package main import ( + "encoding/base64" + "encoding/json" "fmt" "sort" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/views" ) // The seat's verbs that read (novox/hq ADR 0239; design 47 G §431, §172, A-24): from G's delivery the gate keeps no merge @@ -111,9 +115,66 @@ func firstOf(a, b string) string { return b } +// DeliveriesFilter narrows `deliveries` (design 47 H2 §811): state (open or ended), repository, group, the module a record +// moves, the outcome it ended with and the time it opened at or after; All lists the ended ones of every day kept; Next is +// the cursor the previous page answered. +type DeliveriesFilter struct { + State, Repository, Group, Module, Outcome string + Since time.Time + All bool + Next string +} + +// movesModule is whether one of the record's entries is the module's. +func movesModule(r *Record, module string) bool { + for _, e := range r.Entries { + if e.Module == module { + return true + } + } + return false +} + +// endedWith is whether the record ended with the outcome: its why, or the why's first word before a colon ("failed" for +// "failed: "). +func endedWith(r *Record, outcome string) bool { + if r.Ended == nil { + return false + } + return r.Ended.Why == outcome || strings.HasPrefix(r.Ended.Why, outcome+":") +} + +// deliveriesCursor is the opaque cursor of a page of `deliveries` (§779): the place of the last record it answered in the +// order open first, then the newest. +func deliveriesCursor(v recordView) string { + return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%t|%d|%s", v.Record.Ended == nil, v.Record.Opened.UnixNano(), v.Instance))) +} + +// beforeOrAt is whether v comes at or before the place cursor names, in the order of `deliveries`. +func beforeOrAt(v recordView, cursor string) (bool, error) { + raw, err := base64.RawURLEncoding.DecodeString(cursor) + parts := strings.SplitN(string(raw), "|", 3) + if err != nil || len(parts) != 3 { + return false, fmt.Errorf("deliveries: %q is no cursor this verb answered", cursor) + } + var opened int64 + if _, err := fmt.Sscanf(parts[1], "%d", &opened); err != nil { + return false, fmt.Errorf("deliveries: %q is no cursor this verb answered", cursor) + } + open := v.Record.Ended == nil + switch cOpen := parts[0] == "true"; { + case open != cOpen: + return open, nil + case v.Record.Opened.UnixNano() != opened: + return v.Record.Opened.UnixNano() > opened, nil + } + return v.Instance >= parts[2], nil +} + // Deliveries is the `deliveries` verb: every record kept as it is kept, the open ones first, then the newest; the ended ones -// of the last thirty days (all: every one kept), narrowed by state (open or ended), repository or group. -func (h *Holder) Deliveries(state, repository, group string, all bool) (map[string]any, error) { +// of the last thirty days (all: every one kept), narrowed by the filter while it reads, at most views.PageBytes a page +// with capped, total and the next page's cursor (design 47 H2 §811, §779), never refused as too big. +func (h *Holder) Deliveries(f DeliveriesFilter) (map[string]any, error) { kept, err := h.keptRecords() if err != nil { return nil, err @@ -121,10 +182,11 @@ func (h *Holder) Deliveries(state, repository, group string, all bool) (map[stri now := h.Now() var out []recordView for _, r := range kept { - if state != "" && state != stateOf(r) || repository != "" && !holdsRepository(r, repository) || group != "" && groupOf(r) != group { - continue - } - if r.Ended != nil && !all && now.Sub(r.Ended.At) > listedFor { + switch { + case f.State != "" && f.State != stateOf(r), f.Repository != "" && !holdsRepository(r, f.Repository), + f.Group != "" && groupOf(r) != f.Group, f.Module != "" && !movesModule(r, f.Module), + f.Outcome != "" && !endedWith(r, f.Outcome), !f.Since.IsZero() && r.Opened.Before(f.Since), + r.Ended != nil && !f.All && now.Sub(r.Ended.At) > listedFor: continue } out = append(out, viewOf(r)) @@ -133,12 +195,30 @@ func (h *Holder) Deliveries(state, repository, group string, all bool) (map[stri if (out[i].Record.Ended == nil) != (out[j].Record.Ended == nil) { return out[i].Record.Ended == nil } - return out[i].Record.Opened.After(out[j].Record.Opened) + if !out[i].Record.Opened.Equal(out[j].Record.Opened) { + return out[i].Record.Opened.After(out[j].Record.Opened) + } + return out[i].Instance > out[j].Instance }) - if out == nil { - out = []recordView{} + page, size, capped, next := []recordView{}, 0, false, "" + for _, v := range out { + if f.Next != "" { + at, err := beforeOrAt(v, f.Next) + if err != nil { + return nil, err + } + if at { + continue + } + } + raw, _ := json.Marshal(v) + if size+len(raw) > views.PageBytes && len(page) > 0 { + capped, next = true, deliveriesCursor(page[len(page)-1]) + break + } + page, size = append(page, v), size+len(raw) } - return map[string]any{"deliveries": out}, nil + return map[string]any{"deliveries": page, "capped": capped, "total": len(out), "next": next}, nil } func (h *Holder) keptRecords() ([]*Record, error) {