jschoubben
e2801773c3
A periodic report about an identical earlier send names the same digest and can arrive after a later send the machine never received, so the digest and the time read it as applied. The digest-and-time rule stays for engines that report no order. Tests now hold the digest condition and the sequence one.
349 lines
14 KiB
Go
349 lines
14 KiB
Go
package inventory
|
|
|
|
import (
|
|
"errors"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration
|
|
// newer in sequence than every other named four fewer modules than the assignments held, a machine applied
|
|
// it, and nothing the mesh kept could say who sent it or from what view.
|
|
|
|
func generationNow(t *testing.T, inv *Inventory) int64 {
|
|
t.Helper()
|
|
g, err := inv.AssignmentGeneration(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return g
|
|
}
|
|
|
|
// **Every change to what is assigned where raises the generation, in its own transaction** — an
|
|
// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and
|
|
// a repeated assignment, which changes nothing, does not.
|
|
func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) {
|
|
inv, node := aNodeWithModules(t, "postgres", "web")
|
|
ctx := t.Context()
|
|
start := generationNow(t, inv)
|
|
if start < 1 {
|
|
t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start)
|
|
}
|
|
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assigned := generationNow(t, inv)
|
|
if assigned <= start {
|
|
t.Fatalf("an assignment left the generation at %d", assigned)
|
|
}
|
|
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if again := generationNow(t, inv); again != assigned {
|
|
t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again)
|
|
}
|
|
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
unassigned := generationNow(t, inv)
|
|
if unassigned <= assigned {
|
|
t.Fatalf("an unassignment left the generation at %d", unassigned)
|
|
}
|
|
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := generationNow(t, inv)
|
|
if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if removed := generationNow(t, inv); removed <= before {
|
|
t.Errorf("a machine's removal took its assignments and left the generation at %d", removed)
|
|
}
|
|
}
|
|
|
|
// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter
|
|
// is behind it** — a store put back from a backup — so the next send is not refused for ever.
|
|
func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) {
|
|
inv, _ := aNodeWithModules(t)
|
|
now := generationNow(t, inv)
|
|
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 {
|
|
t.Fatalf("raised past %d to %d (%v)", now+40, raised, err)
|
|
}
|
|
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 {
|
|
t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err)
|
|
}
|
|
}
|
|
|
|
// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it
|
|
// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation
|
|
// written with the digest, and a refusal is kept on the send it refused, naming its sender.
|
|
func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)",
|
|
Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}}
|
|
if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor",
|
|
Generation: 38, Digest: "d12", Modules: []string{"docker"}}
|
|
if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
last, found, err := inv.LastSend(ctx, node)
|
|
if err != nil || !found {
|
|
t.Fatalf("the last send is not kept: %v", err)
|
|
}
|
|
if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded ||
|
|
!slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() {
|
|
t.Fatalf("the last send reads %+v", last)
|
|
}
|
|
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 {
|
|
t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err)
|
|
}
|
|
|
|
refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0)
|
|
if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 ||
|
|
refused.CounterRaisedTo != 0 {
|
|
t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err)
|
|
}
|
|
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
|
|
if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 {
|
|
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
|
|
}
|
|
if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found {
|
|
t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err)
|
|
}
|
|
}
|
|
|
|
// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it;
|
|
// it is not the machine's last send, and it says what the counter was raised to.
|
|
func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41,
|
|
Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 ||
|
|
kept.NodeName != node {
|
|
t.Fatalf("the refusal reads %+v", kept)
|
|
}
|
|
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
|
|
if err != nil || len(since) != 1 || since[0].Sequence != 99 {
|
|
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
|
|
}
|
|
if _, found, err := inv.LastSend(ctx, node); err != nil || found {
|
|
t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err)
|
|
}
|
|
}
|
|
|
|
// **The machine's record and the send's are written together, and a send record that cannot be written
|
|
// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a
|
|
// send that is away must not read as failed, nor the machine as behind.
|
|
func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A NUL byte is text PostgreSQL refuses: the send's own row cannot be written.
|
|
broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"}
|
|
err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken)
|
|
var unrecorded *SendNotRecordedError
|
|
if !errors.As(err, &unrecorded) {
|
|
t.Fatalf("a send record that could not be written read as %v", err)
|
|
}
|
|
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 {
|
|
t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err)
|
|
}
|
|
if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" {
|
|
t.Fatalf("the machine's digest was not written: %q (%v)", sent, err)
|
|
}
|
|
}
|
|
|
|
// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN).
|
|
func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) {
|
|
inv, node := aNodeWithModules(t, "web")
|
|
ctx := t.Context()
|
|
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := generationNow(t, inv)
|
|
if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if after := generationNow(t, inv); after != before {
|
|
t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after)
|
|
}
|
|
}
|
|
|
|
// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer.
|
|
func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
record, err := inv.NodeByName(t.Context(), node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, reads := range []bool{false, true, false} {
|
|
if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads {
|
|
t.Fatalf("recorded %v, read %v (%v)", reads, got, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
|
|
// words, or not answered yet — read from the machine's report only when that report is about the send's
|
|
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
|
|
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
|
|
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
send := func(sequence int64, digest string) {
|
|
t.Helper()
|
|
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
|
|
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
|
|
Modules: []string{"docker", "sudo"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
report := func(d Doing) {
|
|
t.Helper()
|
|
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
answer := func() SendAnswer {
|
|
t.Helper()
|
|
last, found, err := inv.LastSend(ctx, node)
|
|
if err != nil || !found {
|
|
t.Fatalf("the last send is not read back: %v, %v", found, err)
|
|
}
|
|
return last.Answer
|
|
}
|
|
|
|
send(11, "d11")
|
|
if a := answer(); a.Outcome != "" || a.At != nil {
|
|
t.Fatalf("a send not reported on reads as answered: %+v", a)
|
|
}
|
|
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
|
|
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
|
|
t.Fatalf("a send the machine reported applying reads %+v", a)
|
|
}
|
|
|
|
send(12, "d12")
|
|
if a := answer(); a.Outcome != "" {
|
|
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
|
|
}
|
|
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
|
|
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
|
|
t.Fatalf("a send the machine refused reads %+v", a)
|
|
}
|
|
|
|
// Sent again a declaration it reported applying before the send: not answered until it reports again.
|
|
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
|
|
send(13, "d12")
|
|
if a := answer(); a.Outcome != "" {
|
|
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
|
|
}
|
|
|
|
all, err := inv.LastSends(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
|
|
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
|
|
t.Fatalf("every machine's last send reads %+v", all)
|
|
}
|
|
if _, found := all["laptop"]; found {
|
|
t.Error("a machine never sent anything has a last send")
|
|
}
|
|
}
|
|
|
|
// **A report about another declaration, or another send of the same one, is not the last send's answer**
|
|
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
|
|
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
|
|
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
|
|
// the report's epoch and sequence say which send it is about.
|
|
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
send := func(sequence int64, digest string) {
|
|
t.Helper()
|
|
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
|
|
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
ordered := func(epoch, sequence int64, digest string) {
|
|
t.Helper()
|
|
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
|
|
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
answer := func() string {
|
|
t.Helper()
|
|
last, found, err := inv.LastSend(ctx, node)
|
|
if err != nil || !found {
|
|
t.Fatalf("the last send is not read back: %v, %v", found, err)
|
|
}
|
|
return last.Answer.Outcome
|
|
}
|
|
|
|
// An engine that reports no order: the digest decides.
|
|
send(11, "d11")
|
|
send(12, "d12")
|
|
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a := answer(); a != "" {
|
|
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
|
|
}
|
|
|
|
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
|
|
send(20, "d20")
|
|
ordered(57, 20, "d20")
|
|
if a := answer(); a != OutcomeApplied {
|
|
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
|
|
}
|
|
send(21, "d20")
|
|
ordered(57, 20, "d20")
|
|
if a := answer(); a != "" {
|
|
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
|
|
}
|
|
ordered(56, 21, "d20")
|
|
if a := answer(); a != "" {
|
|
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
|
|
}
|
|
ordered(57, 21, "d20")
|
|
if a := answer(); a != OutcomeApplied {
|
|
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
|
|
}
|
|
}
|