package main import ( "context" "errors" "strings" "testing" "time" ) // The review of mesh-catalog#211: every way a review can fail is bounded, per record, and loud. // A needs-operator record whose ask cannot be made is reviewed again at most MostReviewTries times, however long // the clock runs: the tries are carried, not started again at each failure. func TestAnAskThatCannotBeMadeIsTriedABoundedNumberOfTimes(t *testing.T) { w := imported(t, newMemStore()) w.k.Asks = nil n := proposed(t, w, "author@g14", true) for i := 0; i < 6*48; i++ { // two days, every ten minutes w.reviewAll(t) w.clock = w.clock.Add(10 * time.Minute) } e := mustGet(t, w.k.Store, Key{KindDecision, n}) if len(w.judge.reviews) != MostReviewTries || e.Current.Judging == nil || e.Current.Judging.Tries != MostReviewTries || len(e.Current.Reviews) > MostReviewTries || e.Current.Status != StatusProposed { t.Fatalf("%d judgements, judging %+v, %d reviews kept", len(w.judge.reviews), e.Current.Judging, len(e.Current.Reviews)) } } // failingSaves is a store whose writes of one record fail, as a database refusing them would. type failingSaves struct { *memStore key Key fail bool } func (f *failingSaves) Save(ctx context.Context, v Version, current, clearPending bool, revision int64) error { if f.fail && v.Record != nil && v.Record.Key() == f.key { return errors.New("the database refused the write") } return f.memStore.Save(ctx, v, current, clearPending, revision) } // A record whose review cannot be kept does not block the others, and is tried a bounded number of times. func TestARecordWhoseReviewCannotBeKeptBlocksNoOtherAndIsTriedBoundedly(t *testing.T) { s := &failingSaves{memStore: newMemStore()} w := imported(t, s) a := proposed(t, w, "author@g14", false) b := proposed(t, w, "author@g14", false) s.key, s.fail = Key{KindDecision, a}, true errs := 0 for i := 0; i < 6*48; i++ { for j := 0; j < 5; j++ { said, err := w.k.ReviewStep() if err != nil { errs++ continue } if said == "" { break } } w.clock = w.clock.Add(10 * time.Minute) } if got := mustGet(t, s, Key{KindDecision, b}).Current.Status; got != StatusAccepted { t.Fatalf("ADR %04d stayed %s behind a record whose review could not be kept", b, got) } if errs == 0 || errs > MostReviewTries { t.Fatalf("the failing record was tried %d times; at most %d", errs, MostReviewTries) } if mustGet(t, s, Key{KindDecision, a}).Current.Status != StatusProposed { t.Fatal("accepted without its review kept") } } // A record that ran out of review tries waits on its author, not on the module: it is told to the operator and leaves // the module healthy, so it never holds the module's own rollouts (novox/hq issue 508). A notice that could not be // sent is sent again at the next step, since health no longer stands for it. func TestARecordThatCouldNotBeReviewedLeavesTheModuleHealthyAndIsTold(t *testing.T) { w := imported(t, newMemStore()) n := proposed(t, w, "author@g14", false) w.judge.reviewFail = true notify := w.k.Notify failing := true w.k.Notify = func(key, summary, headline, explanation, resolved string) error { if failing { failing = false return errors.New("the operator's channel did not answer") } return notify(key, summary, headline, explanation, resolved) } for i := 0; i < MostReviewTries+1; i++ { w.reviewAll(t) if h := w.k.Health(nil); !h.Healthy { t.Fatalf("unhealthy after try %d: %s", i+1, h.Why) } w.clock = w.clock.Add(time.Hour) } w.reviewAll(t) h := w.k.Health(nil) if !h.Healthy || h.State != "healthy" || !strings.Contains(h.Why, itoaKey(n)) || !strings.Contains(h.Why, "author") { t.Fatalf("not healthy, or the record waiting on its author not named: %+v", h) } if len(w.told) != 1 || !strings.Contains(w.told[0], itoaKey(n)) || !strings.Contains(w.told[0], "never reviewed") { t.Fatalf("told %q", w.told) } } // A progressive insight waiting for its review is replaced only by its author, proven. func TestAPendingInsightIsReplacedOnlyByItsAuthor(t *testing.T) { w := imported(t, newMemStore()) n := w.reserved(t, "author@g14") w.accepted(t, "author@g14", valid(n)) p := valid(n) p.Sections[2].Text = "Merges land sooner, by about a minute." p.Insight = "it said only that merges land sooner" if _, err := w.k.Propose("corrector@ace", p); err != nil { t.Fatal(err) } q := valid(n) q.Sections[2].Text = "Merges land later." q.Insight = "it said merges land sooner" for _, other := range []string{"author@g14", "other@novox", Unproven} { if _, err := w.k.Propose(other, q); err == nil || !strings.Contains(err.Error(), "only its author") { t.Fatalf("a pending insight replaced by %s: %v", other, err) } } if e := mustGet(t, w.k.Store, Key{KindDecision, n}); e.Pending.Section("Consequences") != p.Sections[2].Text { t.Fatalf("the pending insight was replaced: %q", e.Pending.Section("Consequences")) } if _, err := w.k.Propose("corrector@ace", q); err != nil { t.Fatalf("its author's revision: %v", err) } }