mesh-catalog/modules/mesh-knowledge/cmd/mesh-knowledge/review_bounds_test.go

package main

import (
	"context"
	"errors"
	"strings"
	"testing"
	"time"
)

// The review of mesh-catalog#211: every way a review can fail is bounded, per record, and loud.

// A needs-operator record whose ask cannot be made is reviewed again at most MostReviewTries times, however long
// the clock runs: the tries are carried, not started again at each failure.
func TestAnAskThatCannotBeMadeIsTriedABoundedNumberOfTimes(t *testing.T) {
	w := imported(t, newMemStore())
	w.k.Asks = nil
	n := proposed(t, w, "author@node-b", true)
	for i := 0; i < 6*48; i++ { // two days, every ten minutes
		w.reviewAll(t)
		w.clock = w.clock.Add(10 * time.Minute)
	}
	e := mustGet(t, w.k.Store, Key{KindDecision, n})
	if len(w.judge.reviews) != MostReviewTries || e.Current.Judging == nil || e.Current.Judging.Tries != MostReviewTries ||
		len(e.Current.Reviews) > MostReviewTries || e.Current.Status != StatusProposed {
		t.Fatalf("%d judgements, judging %+v, %d reviews kept", len(w.judge.reviews), e.Current.Judging, len(e.Current.Reviews))
	}
}

// failingSaves is a store whose writes of one record fail, as a database refusing them would.
type failingSaves struct {
	*memStore
	key  Key
	fail bool
}

func (f *failingSaves) Save(ctx context.Context, v Version, current, clearPending bool, revision int64) error {
	if f.fail && v.Record != nil && v.Record.Key() == f.key {
		return errors.New("the database refused the write")
	}
	return f.memStore.Save(ctx, v, current, clearPending, revision)
}

// A record whose review cannot be kept does not block the others, and is tried a bounded number of times.
func TestARecordWhoseReviewCannotBeKeptBlocksNoOtherAndIsTriedBoundedly(t *testing.T) {
	s := &failingSaves{memStore: newMemStore()}
	w := imported(t, s)
	a := proposed(t, w, "author@node-b", false)
	b := proposed(t, w, "author@node-b", false)
	s.key, s.fail = Key{KindDecision, a}, true
	errs := 0
	for i := 0; i < 6*48; i++ {
		for j := 0; j < 5; j++ {
			said, err := w.k.ReviewStep()
			if err != nil {
				errs++
				continue
			}
			if said == "" {
				break
			}
		}
		w.clock = w.clock.Add(10 * time.Minute)
	}
	if got := mustGet(t, s, Key{KindDecision, b}).Current.Status; got != StatusAccepted {
		t.Fatalf("ADR %04d stayed %s behind a record whose review could not be kept", b, got)
	}
	if errs == 0 || errs > MostReviewTries {
		t.Fatalf("the failing record was tried %d times; at most %d", errs, MostReviewTries)
	}
	if mustGet(t, s, Key{KindDecision, a}).Current.Status != StatusProposed {
		t.Fatal("accepted without its review kept")
	}
}

// A record that ran out of review tries waits on its author, not on the module: it is told to the operator and leaves
// the module healthy, so it never holds the module's own rollouts (novox/hq issue 508). A notice that could not be
// sent is sent again at the next step, since health no longer stands for it.
func TestARecordThatCouldNotBeReviewedLeavesTheModuleHealthyAndIsTold(t *testing.T) {
	w := imported(t, newMemStore())
	n := proposed(t, w, "author@node-b", false)
	w.judge.reviewFail = true
	notify := w.k.Notify
	failing := true
	w.k.Notify = func(key, summary, headline, explanation, resolved string) error {
		if failing {
			failing = false
			return errors.New("the operator's channel did not answer")
		}
		return notify(key, summary, headline, explanation, resolved)
	}
	for i := 0; i < MostReviewTries+1; i++ {
		w.reviewAll(t)
		if h := w.k.Health(nil); !h.Healthy {
			t.Fatalf("unhealthy after try %d: %s", i+1, h.Why)
		}
		w.clock = w.clock.Add(time.Hour)
	}
	w.reviewAll(t)
	h := w.k.Health(nil)
	if !h.Healthy || h.State != "healthy" || !strings.Contains(h.Why, itoaKey(n)) || !strings.Contains(h.Why, "author") {
		t.Fatalf("not healthy, or the record waiting on its author not named: %+v", h)
	}
	if len(w.told) != 1 || !strings.Contains(w.told[0], itoaKey(n)) || !strings.Contains(w.told[0], "never reviewed") {
		t.Fatalf("told %q", w.told)
	}
}

// A progressive insight waiting for its review is replaced only by its author, proven.
func TestAPendingInsightIsReplacedOnlyByItsAuthor(t *testing.T) {
	w := imported(t, newMemStore())
	n := w.reserved(t, "author@node-b")
	w.accepted(t, "author@node-b", valid(n))
	p := valid(n)
	p.Sections[2].Text = "Merges land sooner, by about a minute."
	p.Insight = "it said only that merges land sooner"
	if _, err := w.k.Propose("corrector@node-a", p); err != nil {
		t.Fatal(err)
	}
	q := valid(n)
	q.Sections[2].Text = "Merges land later."
	q.Insight = "it said merges land sooner"
	for _, other := range []string{"author@node-b", "other@node-c", Unproven} {
		if _, err := w.k.Propose(other, q); err == nil || !strings.Contains(err.Error(), "only its author") {
			t.Fatalf("a pending insight replaced by %s: %v", other, err)
		}
	}
	if e := mustGet(t, w.k.Store, Key{KindDecision, n}); e.Pending.Section("Consequences") != p.Sections[2].Text {
		t.Fatalf("the pending insight was replaced: %q", e.Pending.Section("Consequences"))
	}
	if _, err := w.k.Propose("corrector@node-a", q); err != nil {
		t.Fatalf("its author's revision: %v", err)
	}
}