package link_test import ( "context" "testing" "time" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A report kept by its order (novox/hq to-be 45 §6, ADR 0227 rule 2): the contract test of the report, // the message kind the controller consumes from every machine. Deliver n, then n−1: refused and counted. // Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing // for an ordered report; an unordered one, from an older node-engine, is judged by it as before. func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) { t.Helper() inv := inventory.ForTest(t) node, err := inv.AddNode(context.Background(), "home-server") if err != nil { t.Fatal(err) } return inv, link.Enrolment{Inventory: inv}, node.ID } func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report { return link.Report{Node: "home-server", Declared: declared, Applied: applied, Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report} } // Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12. func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) { inv, heard, id := anOrderedMachine(t) ctx := context.Background() before := countFor(link.WriterNodeEngine("home-server")) if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { t.Fatal(err) } if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil { t.Fatal(err) } doing, _, err := inv.DoingOf(ctx, "home-server") if err != nil || doing.Declared != "d12" || doing.Applied != 2 { t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err) } if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 { t.Fatalf("the refusal was counted %d times, want once", got) } kept, err := inv.KeptOrder(ctx, id) if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) { t.Fatalf("the order kept is %+v (%v)", kept, err) } } func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) { inv, heard, _ := anOrderedMachine(t) ctx := context.Background() if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { t.Fatal(err) } failed := ordered("d12", 57, 12, 40) failed.Failed = map[string]string{"b": "it failed a moment before it applied"} if _, err := heard.Heard(ctx, failed); err != nil { t.Fatal(err) } doing, _, _ := inv.DoingOf(ctx, "home-server") if doing.Outcome != inventory.OutcomeApplied { t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing) } // A newer report of it — the next reconcile — is kept. again := ordered("d12", 57, 12, 42) again.Failed = map[string]string{"b": "it failed since"} if _, err := heard.Heard(ctx, again); err != nil { t.Fatal(err) } if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed { t.Fatalf("a newer report of the same declaration was not kept: %+v", doing) } } // An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12 // is kept although the mesh has sent 13 since — it is still the newest account of the machine. func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) { inv, heard, id := anOrderedMachine(t) ctx := context.Background() if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { t.Fatal(err) } if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { t.Fatal(err) } if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" { t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing) } // And the machine reads an epoch: it orders its reports. if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads { t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err) } } // A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the // order kept is cleared, and it is no longer sent an epoch. func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { inv, heard, id := anOrderedMachine(t) ctx := context.Background() if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { t.Fatal(err) } if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { t.Fatal(err) } if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil { t.Fatal(err) } if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" { t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing) } if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) { t.Fatalf("an unordered account left the order kept: %+v", kept) } if reads, _ := inv.ReadsEpoch(ctx, id); reads { t.Fatal("a node-engine that orders nothing is still sent an epoch") } // The next ordered account, whatever its numbers, has nothing to be older than. if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil { t.Fatal(err) } if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" { t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing) } } // A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose // own report was lost is still counted from the machine's own tally. func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) { count := link.NewRefusalCount() now := time.Now() refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11}, OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3} if !refusal.StaleRefusalOf() { t.Fatal("a refusal naming the order it holds is not stale") } count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now}) count.Lifetime("anchor", refusal.RefusedOlder, now) count.Lifetime("anchor", 5, now) // two more refused, their reports lost got := count.Within(now.Add(-time.Minute)) if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" || got[1].Count != 1 || got[1].Receivers[0] != "anchor" { t.Fatalf("the refusals by writer are %+v", got) } } // countFor is how many refusals of a writer this process has heard in the last minute. func countFor(writer string) int { for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) { if w.Writer == writer { return w.Count } } return 0 }