mesh-controller/internal/link/heard_order_test.go

package link_test

import (
	"context"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A report kept by its order (novox/hq to-be 45 §6, ADR 0227 rule 2): the contract test of the report,
// the message kind the controller consumes from every machine. Deliver n, then n−1: refused and counted.
// Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing
// for an ordered report; an unordered one, from an older node-engine, is judged by it as before.

func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) {
	t.Helper()
	inv := inventory.ForTest(t)
	node, err := inv.AddNode(context.Background(), "home-server")
	if err != nil {
		t.Fatal(err)
	}
	return inv, link.Enrolment{Inventory: inv}, node.ID
}

func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report {
	return link.Report{Node: "home-server", Declared: declared, Applied: applied,
		Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report}
}

// Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12.
func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) {
	inv, heard, id := anOrderedMachine(t)
	ctx := context.Background()
	before := countFor(link.WriterNodeEngine("home-server"))
	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
		t.Fatal(err)
	}
	if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil {
		t.Fatal(err)
	}
	doing, _, err := inv.DoingOf(ctx, "home-server")
	if err != nil || doing.Declared != "d12" || doing.Applied != 2 {
		t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err)
	}
	if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 {
		t.Fatalf("the refusal was counted %d times, want once", got)
	}
	kept, err := inv.KeptOrder(ctx, id)
	if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) {
		t.Fatalf("the order kept is %+v (%v)", kept, err)
	}
}

func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) {
	inv, heard, _ := anOrderedMachine(t)
	ctx := context.Background()
	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
		t.Fatal(err)
	}
	failed := ordered("d12", 57, 12, 40)
	failed.Failed = map[string]string{"b": "it failed a moment before it applied"}
	if _, err := heard.Heard(ctx, failed); err != nil {
		t.Fatal(err)
	}
	doing, _, _ := inv.DoingOf(ctx, "home-server")
	if doing.Outcome != inventory.OutcomeApplied {
		t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing)
	}
	// A newer report of it — the next reconcile — is kept.
	again := ordered("d12", 57, 12, 42)
	again.Failed = map[string]string{"b": "it failed since"}
	if _, err := heard.Heard(ctx, again); err != nil {
		t.Fatal(err)
	}
	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed {
		t.Fatalf("a newer report of the same declaration was not kept: %+v", doing)
	}
}

// An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12
// is kept although the mesh has sent 13 since — it is still the newest account of the machine.
func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) {
	inv, heard, id := anOrderedMachine(t)
	ctx := context.Background()
	if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
		t.Fatal(err)
	}
	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
		t.Fatal(err)
	}
	if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" {
		t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing)
	}
	// And the machine reads an epoch: it orders its reports.
	if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads {
		t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err)
	}
}

// A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the
// order kept is cleared, and it is no longer sent an epoch.
func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) {
	inv, heard, id := anOrderedMachine(t)
	ctx := context.Background()
	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
		t.Fatal(err)
	}
	if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil {
		t.Fatal(err)
	}
	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" {
		t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing)
	}
	if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) {
		t.Fatalf("an unordered account left the order kept: %+v", kept)
	}
	if reads, _ := inv.ReadsEpoch(ctx, id); reads {
		t.Fatal("a node-engine that orders nothing is still sent an epoch")
	}
	// The next ordered account, whatever its numbers, has nothing to be older than.
	if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil {
		t.Fatal(err)
	}
	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" {
		t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing)
	}
}

// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose
// own report was lost is still counted from the machine's own tally.
func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) {
	count := link.NewRefusalCount()
	now := time.Now()
	refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11},
		OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3}
	if !refusal.StaleRefusalOf() {
		t.Fatal("a refusal naming the order it holds is not stale")
	}
	count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it
	count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now})
	count.Lifetime("anchor", refusal.RefusedOlder, now)
	count.Lifetime("anchor", 5, now) // two more refused, their reports lost
	got := count.Within(now.Add(-time.Minute))
	if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" ||
		got[1].Count != 1 || got[1].Receivers[0] != "anchor" {
		t.Fatalf("the refusals by writer are %+v", got)
	}
}

// countFor is how many refusals of a writer this process has heard in the last minute.
func countFor(writer string) int {
	for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) {
		if w.Writer == writer {
			return w.Count
		}
	}
	return 0
}