diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index d55ddff0..378ae6c7 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -222,6 +222,12 @@ type Indexer struct { Pending Store // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error + // ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker, + // A-248); nil validates nothing, and a revision is taken in as before. + ValidationAsks ValidationAsks + // ReadStepOutput reads a step-ended's output object, where the build seat keeps an answer too large for the event + // (A-247); nil reads none, and such an answer could not validate. + ReadStepOutput func(ctx context.Context, object string) ([]byte, error) // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. Runs Store runMu sync.Mutex // one run event at a time (runcandidates.go) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index ae1aee31..15a81ecf 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -98,6 +98,8 @@ type Pending struct { // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), // until room is said (RoomAgain). Held string `json:"held,omitempty"` + // Validation is the revision's validation (design 47 G §713, A-247), on the revision's newest value alone. + Validation *takein.Validation `json:"validation,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -773,6 +775,9 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if len(lost) > 0 { return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread) } + if wait, words, err := ix.validationGate(ctx, ready, own); wait || err != nil { + return Applied{}, words, err + } names, dry, err := ix.computeOver(ctx, ready) if err != nil { return Applied{}, "", err @@ -904,6 +909,7 @@ func (ix *Indexer) dry() *dryRun { x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains, ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, + ValidationAsks: nil, ReadStepOutput: nil, // a dry run asks no validation (§713) // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. @@ -1332,6 +1338,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { if e.Value.Refused != "" { pw.Refused = e.Value.Refused } + if e.Value.Validation != nil { + pw.Validation = e.Value.Validation + } if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index a3e072a5..04c22a84 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Runs", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "InputObjects", "BuildEdges", "Pins", "PinsPending"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Runs", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "InputObjects", "BuildEdges", "Pins", "PinsPending", "ValidationAsks", "ReadStepOutput"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/validate.go b/modules/mesh-delivery-indexer/internal/index/validate.go index 103c3e6a..0fbf3bdc 100644 --- a/modules/mesh-delivery-indexer/internal/index/validate.go +++ b/modules/mesh-delivery-indexer/internal/index/validate.go @@ -25,9 +25,9 @@ const ( // validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's // own code, so no repository or module name is fixed here (ADR 0666 §5). validationProgram = "cmd/mesh-judge" - // validationAsker is the asker the ask is put under. A-232 names the indexer's seat name mesh-catalog; the ask code still + // ValidationAsker is the asker the ask is put under (buildrun.Asker.Name, A-248). A-232 names the indexer's seat name mesh-catalog; the ask code still // names askers by their nicknames until the asker rename after the join (A-163), so this is `indexer` until then. - validationAsker = "indexer" + ValidationAsker = "indexer" // validationCatalogSeat is the seat the indexer entry claims. validationCatalogSeat = "mesh-catalog" // The cases of the request, as the manifest validator names them. diff --git a/modules/mesh-delivery-indexer/internal/index/validation_wait.go b/modules/mesh-delivery-indexer/internal/index/validation_wait.go new file mode 100644 index 00000000..f5fc3f5b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/validation_wait.go @@ -0,0 +1,266 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" +) + +// The pending revision's wait for its validation (design 47 G §713, A-232, A-247): the take-in puts the run ask of §712, writes +// `validation` on the revision's newest pending value and ends there; the build seat's step-ended of that ask, heard through the +// indexer's own durable consumer, resumes it: accepted takes the revision in as before, refused keeps it pending with `refused`, +// and an end that could not validate keeps it pending with `failed`, asked again by a later merge of the branch or by hand. + +// ValidationAsks puts a run ask on the build seat (buildrun.Asker). +type ValidationAsks interface { + Put(ctx context.Context, a buildstep.StepAsk) error +} + +// newestOfRevision is the index in ready of the revision's newest merge, the value that carries `validation` (A-247). +func newestOfRevision(ready []PendingEntry) int { + n := 0 + for i, e := range ready { + if e.Value.Seq > ready[n].Value.Seq { + n = i + } + } + return n +} + +func branchOfRevision(ready []PendingEntry) string { + if ref := ready[newestOfRevision(ready)].Value.Ref; ref != nil && ref.Branch != "" { + return ref.Branch + } + return TrunkBranch +} + +// validationGate says whether the ready revision waits for its validation (§713): it goes on to the take-in once its newest +// value's validation was accepted at that value's sequence; it waits while its ask is out, while it is refused, and while it +// could not validate; otherwise it puts a new ask, the next try, and waits. Without an asker nothing is validated. +func (ix *Indexer) validationGate(ctx context.Context, ready []PendingEntry, own string) (bool, string, error) { + if ix.ValidationAsks == nil { + return false, "", nil + } + newest := ready[newestOfRevision(ready)] + seq := newest.Value.Seq + if v := newest.Value.Validation; v != nil && v.Seq == seq { + switch { + case v.Accepted == seq: + return false, "", nil + case v.Failed != "": + return true, fmt.Sprintf("the validation of the revision at %d could not validate: %s", seq, v.Failed), nil + case newest.Value.Refused != "": + return true, fmt.Sprintf("the revision at %d is refused: %s", seq, newest.Value.Refused), nil + case v.Ask != "": + return true, fmt.Sprintf("the revision at %d waits for its validation, the ask %s", seq, v.Ask), nil + } + } + return true, "", ix.askValidation(ctx, ready, own) +} + +// askValidation puts the revision's validation ask (§712) and writes it on its newest value, the try after every earlier try +// of the revision; an older value's validation goes (A-247: on the newest alone). A request or job that cannot be made could not +// validate: it is written as `failed`, said, and no ask is put. +func (ix *Indexer) askValidation(ctx context.Context, ready []PendingEntry, own string) error { + n := newestOfRevision(ready) + newest := ready[n] + try := 0 + for _, e := range ready { + if v := e.Value.Validation; v != nil && v.Try > try { + try = v.Try + } + } + v := &takein.Validation{Seq: newest.Value.Seq, Try: try + 1, Asked: ix.now()} + branch := branchOfRevision(ready) + ask := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: ValidationAsker, At: ix.now().UTC()} + req, entry, err := ix.validationRequest(ctx, branch, ready) + var raw []byte + if err == nil { + raw, err = json.Marshal(req) + } + var job buildstep.RunJob + if err == nil { + job, err = validationJob(entry, revisionHead(ready, entry.Repository, entry.Commit), branch, v.Seq, raw) + } + if err == nil { + v.Input, err = ix.KeepInput(ctx, ask.ID, raw) + } + if err != nil { + v.Failed = "the validation could not be asked: " + err.Error() + } else { + v.Ask, ask.Run = ask.ID, &job + } + if err := ix.writeValidation(ctx, ready, n, v, own); err != nil { + return err + } + if v.Failed != "" { + ix.sayValidation(ctx, ready, v) + return nil + } + if err := ix.ValidationAsks.Put(ctx, ask); err != nil { + return fmt.Errorf("the validation ask %s of the revision at %d could not be put: %w", ask.ID, v.Seq, err) + } + ix.logf("take-in: the revision at %d is validated by the ask %s (try %d); it waits for its end", v.Seq, ask.ID, v.Try) + return nil +} + +// revisionHead is the revision's head of repository: the newest merge of it the revision covers, else the commit its entry stands at. +func revisionHead(ready []PendingEntry, repository, commit string) string { + var seq uint64 + head := commit + for _, e := range ready { + if e.Forge.Merge != "" && strings.EqualFold(e.Forge.Repository, repository) && e.Value.Seq >= seq { + seq, head = e.Value.Seq, e.Forge.Merge + } + } + return head +} + +// writeValidation writes v on ready[n] and takes it from every other value, each by compare-and-set; a new validation clears the +// value's refusal. A value lost under it is an error: the evaluation is made again. +func (ix *Indexer) writeValidation(ctx context.Context, ready []PendingEntry, n int, v *takein.Validation, own string) error { + for i, e := range ready { + if i != n && e.Value.Validation == nil { + continue + } + p := e.Value + p.Validation, p.LastWriter = nil, own + if i == n { + p.Validation, p.Refused = v, "" + } + data, err := encodePending(p) + if err != nil { + return err + } + if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil { + return fmt.Errorf("%s.%s: %w", PendingState, e.Key, err) + } + } + return nil +} + +// sayValidation says the revision's wait in take-in.pending with its refusal or why it could not validate. +func (ix *Indexer) sayValidation(ctx context.Context, ready []PendingEntry, v *takein.Validation) { + t := revisionWait(takein.TakeInPending, ready, "") + t.Failed = v.Failed + t.Refused = ready[newestOfRevision(ready)].Value.Refused + ix.sayTakeIn(ctx, t) +} + +// errNoValidationAnswer is an end whose answer is no verdict of both cases. +var errNoValidationAnswer = errors.New("no answer") + +// HearValidationEnded takes in the end of a validation ask (§713), heard on the build seat's step-ended through the indexer's own +// durable consumer: only an end of the indexer's asks outside a pipeline; an end whose ask no pending value names now is dropped, +// written in the journal. The answer is read from the event's own output or output object (A-247). +func (ix *Indexer) HearValidationEnded(ctx context.Context, e buildstep.StepEnded) error { + if e.Asker != ValidationAsker || e.Pipeline != "" || e.Ask == "" || ix.Pending == nil || !ix.pendingOnBus(ctx) { + return nil + } + own := ix.ownBuildNow(ctx) + ix.pendingMu.Lock() + defer ix.pendingMu.Unlock() + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + return err + } + ready := WaitsOf(entries).Ready + n := -1 + for i, p := range ready { + if p.Value.Validation != nil && p.Value.Validation.Ask == e.Ask { + n = i + } + } + if n < 0 { + ix.logf("take-in: the end of the validation ask %s (%s on %s) is dropped: no pending revision names that ask now", e.Ask, e.Outcome, e.Node) + return nil + } + ready, lost, _, _ := ix.loadNewest(ctx, ready) + if len(lost) > 0 { + return fmt.Errorf("the revision the validation ask %s ended for holds a merge that cannot be read", e.Ask) + } + n = newestOfRevision(ready) + v := *ready[n].Value.Validation + refused, failed := ix.validationAnswer(ctx, ready, e, v) + p := ready[n].Value + switch { + case failed != "": + v.Failed = failed + case refused != "": + p.Refused = refused + default: + v.Accepted = v.Seq + } + p.Validation, p.LastWriter = &v, own + data, err := encodePending(p) + if err != nil { + return err + } + if err := ix.Pending.Update(ctx, ready[n].Key, data, ready[n].Revision); err != nil { + return fmt.Errorf("%s.%s: %w", PendingState, ready[n].Key, err) + } + if v.Accepted == 0 { + ready[n].Value = p + ix.sayValidation(ctx, ready, &v) + ix.logf("take-in: the validation ask %s of the revision at %d ended: refused %q, could not validate %q", e.Ask, v.Seq, refused, failed) + return nil + } + ix.logf("take-in: the validation ask %s accepted the revision at %d", e.Ask, v.Seq) + if own == "" { + return nil // the take-in waits for the build this program runs, as every take-in does + } + _, _, err = ix.evaluate(ctx, own, false) + return err +} + +// validationAnswer reads the end's answer: the refusal of the revision, or why it could not validate (the step did not pass, the +// program exited other than 0, its answer is no verdict of both cases, or its request is no longer the one asked). +func (ix *Indexer) validationAnswer(ctx context.Context, ready []PendingEntry, e buildstep.StepEnded, v takein.Validation) (string, string) { + if e.Outcome != buildstep.OutcomePassed { + return "", strings.TrimSpace(e.Outcome + ": " + e.Said) + } + out := []byte(e.Output) + if len(out) == 0 && e.OutputObject != "" { + if ix.ReadStepOutput == nil { + return "", "the answer is in the output object " + e.OutputObject + ", which this indexer does not read" + } + var err error + if out, err = ix.ReadStepOutput(ctx, e.OutputObject); err != nil { + return "", "the output object " + e.OutputObject + " could not be read: " + err.Error() + } + } + var answer buildstep.RunAnswer + if err := json.Unmarshal(out, &answer); err != nil { + return "", fmt.Sprintf("the run's answer is no run answer: %v", err) + } + if answer.Exit != 0 { + return "", fmt.Sprintf("the manifest validator exited %d: %s", answer.Exit, answer.Said) + } + var verdict composer.Verdict + if err := json.Unmarshal(answer.Output, &verdict); err != nil { + return "", fmt.Sprintf("%v: the manifest validator's output is no verdict: %v", errNoValidationAnswer, err) + } + req, _, err := ix.validationRequest(ctx, branchOfRevision(ready), ready) + var raw []byte + if err == nil { + raw, err = json.Marshal(req) + } + if err != nil { + return "", "the request cannot be made again to read the verdict: " + err.Error() + } + if runDigest(raw) != v.Input { + return "", "the revision's request is no longer the one the ask " + v.Ask + " was asked with" + } + refused, err := ValidationRefusal(req, verdict) + if err != nil { + return "", err.Error() + } + return RefusedWords(refused), "" +} diff --git a/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go b/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go new file mode 100644 index 00000000..5d344a47 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/validation_wait_test.go @@ -0,0 +1,87 @@ +package index + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" +) + +type askRecorder struct{ asks []buildstep.StepAsk } + +func (r *askRecorder) Put(_ context.Context, a buildstep.StepAsk) error { + r.asks = append(r.asks, a) + return nil +} + +// Design 47 G §713, A-247: a pending revision names its ask on its newest value and resumes on that ask's end; an end of a +// replaced ask is dropped; a run that could not validate is no refusal and never passed; the start asks nothing again while +// the ask is out. +func TestAPendingRevisionWaitsForItsValidation(t *testing.T) { + ctx := context.Background() + b := newTakeInBed(t) + asks := &askRecorder{} + b.ix.ValidationAsks, b.ix.Inputs, b.ix.Revisions = asks, newMem(), newMemObjects(0) + // No entry of the revision claims the seat mesh-catalog: the validation could not be asked, so it is failed, said, and + // nothing is taken in. + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "")) + entries := readPending(t, b) + if len(entries) != 1 || entries[0].Value.Validation == nil || !strings.Contains(entries[0].Value.Validation.Failed, "claims the seat mesh-catalog") || len(asks.asks) != 0 { + t.Fatalf("pending %+v, asks %d: want m.2 pending with failed and no ask", entries, len(asks.asks)) + } + if last := b.said[len(b.said)-1]; last.Kind != takein.TakeInPending || last.Failed == "" || last.Refused != "" { + t.Errorf("said %+v, want take-in.pending naming failed, never a refusal", last) + } + if w := PendingWaits(entries); len(w) != 1 || w[0].Validation == nil || w[0].Validation.Try != 1 { + t.Errorf("indexer_pending %+v, want the validation whole", w) + } + // An ask out: the newest value names it; the start asks nothing new while it is out. + loaded, _, _, _ := b.ix.loadNewest(ctx, entries) + req, _, err := b.ix.validationRequest(ctx, TrunkBranch, loaded) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(req) + setValidation := func(v takein.Validation) { + e := readPending(t, b)[0] + p := e.Value + p.Validation = &v + data, _ := encodePending(p) + if err := b.pending.Update(ctx, e.Key, data, e.Revision); err != nil { + t.Fatal(err) + } + } + setValidation(takein.Validation{Ask: "mvl0frv18d1957", Input: runDigest(raw), Seq: 2, Try: 2}) + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(asks.asks) != 0 || len(b.keys()) != 1 { + t.Fatalf("the start: %v, asks %d, pending %v: want it waiting for its ask", err, len(asks.asks), b.keys()) + } + end := func(ask, outcome, said string, answer any) buildstep.StepEnded { + out, _ := json.Marshal(answer) + return buildstep.StepEnded{Ask: ask, ID: ask, Asker: ValidationAsker, Node: "ace", Outcome: outcome, Said: said, Output: string(out)} + } + verdict, _ := json.Marshal(composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change"}, {Name: "trunk"}}}) + accepted := buildstep.RunAnswer{Built: true, Exit: 0, Output: verdict} + // The end of a replaced ask is dropped, written in the journal. + if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1950", buildstep.OutcomePassed, "", accepted)); err != nil || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is dropped") { + t.Fatalf("a replaced ask's end: %v, pending %v", err, b.keys()) + } + // could not run: pending with failed, never passed. + if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1957", buildstep.OutcomeCouldNotRun, "no build node", nil)); err != nil { + t.Fatal(err) + } + if v := readPending(t, b)[0].Value.Validation; v == nil || v.Failed != "could not run: no build node" || v.Accepted != 0 { + t.Fatalf("after could not run: %+v", v) + } + // Asked again (by hand or by a later merge) and accepted: the revision is taken in, and its keys go. + setValidation(takein.Validation{Ask: "mvl0frv18d1958", Input: runDigest(raw), Seq: 2, Try: 3}) + if err := b.ix.HearValidationEnded(ctx, end("mvl0frv18d1958", buildstep.OutcomePassed, "", accepted)); err != nil { + t.Fatal(err) + } + if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') { + t.Fatalf("accepted: pending %v %+v, avahi at %s: want it taken in\n%s", b.keys(), readPending(t, b)[0].Value, b.commit(t, "avahi"), b.logs.String()) + } +} diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index cc3570d5..203872b0 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -141,6 +141,9 @@ type TakeIn struct { // the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for // an indexer whose manifest is accepted, and says why here, as it says what it lost. Refused string `json:"refused,omitempty"` + // Failed is why the revision's validation could not validate (design 47 G §713): the build seat's words, never a refusal + // and never passed; it is asked again by a later merge of the branch and by indexer_validate_again. + Failed string `json:"failed,omitempty"` // Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build // it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown. Unknown string `json:"own-build-unknown,omitempty"` @@ -248,6 +251,21 @@ type PendingWait struct { Unread []string `json:"unread,omitempty"` // Refused is why the manifest of the indexer it names was refused (TakeIn.Refused). Refused string `json:"refused,omitempty"` + // Validation is the revision's validation whole, as its newest pending value keeps it (design 47 G §713, A-247). + Validation *Validation `json:"validation,omitempty"` +} + +// Validation is a pending revision's validation (design 47 G §712, §713, A-232, A-247), kept on the revision's newest pending +// value alone: the run ask's id, when it was asked, its input's digest, the sequence of the newest merge it covers and the try +// number for it, why it could not validate, and the sequence it was accepted at (never asked again once set). +type Validation struct { + Ask string `json:"ask,omitempty"` + Asked time.Time `json:"asked,omitzero"` + Input string `json:"input,omitempty"` + Seq uint64 `json:"seq"` + Try int `json:"try"` + Failed string `json:"failed,omitempty"` + Accepted uint64 `json:"accepted,omitempty"` } // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.