//go:build integration package inventory import ( "errors" "fmt" "strings" "sync" "sync/atomic" "testing" "time" "github.com/novox/mesh-controller/internal/chain" ) // **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state // pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module. func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) { inv, node := aNodeWithModules(t, "web") ctx := t.Context() p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog", Path: "/modules/sensors/"}) if err != nil { t.Fatal(err) } if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" { t.Fatalf("recorded %+v, want id %s", p, want) } kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors") if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() || kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" { t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err) } rows, err := inv.Pending(ctx, time.Time{}) if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" { t.Fatalf("waiting: %+v %v", rows, err) } if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) { t.Fatalf("a second open wait: %v", err) } if _, err := inv.Assign(ctx, node, "web"); err != nil { t.Fatal(err) } if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) { t.Fatalf("a wait for a module that is on the machine: %v", err) } // Written over once it ended: a settled wait is replaced by a new one under a new id. if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok { t.Fatalf("settled %v %v", ok, err) } next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}) if err != nil || next.ID == p.ID { t.Fatalf("a wait after one ended: %+v %v", next, err) } if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); known[p.ID] || !known[next.ID] { t.Fatalf("known %v: the key holds the newer one now", known) } if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 { t.Fatalf("the earlier wait is in the key's history: %d values", len(history)) } if n := rowsInTheTable(t, inv, "pending_assignment"); n != "0" { t.Fatalf("the table took %s pending assignments", n) } } // **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module // runs nowhere, and a module only waiting for a build is not forgotten as assigned. func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) { inv, node := aNodeWithModules(t, "web") ctx := t.Context() if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil { t.Fatal(err) } if on, _ := inv.Assigned(ctx, node); len(on) != 0 { t.Fatalf("assigned %v", on) } if running, _ := inv.Running(ctx, "web"); len(running) != 0 { t.Fatalf("running %v", running) } if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 { t.Fatalf("runs %v", runs) } entries, _ := inv.Catalogued(ctx) for _, e := range entries { if e.Manifest.Module == "web" && len(e.On) != 0 { t.Fatalf("web is on %v", e.On) } } if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) { t.Fatalf("a module only waiting is assigned: %v", err) } } // **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision. func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) { inv, node := aNodeWithModules(t) ctx := t.Context() p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"}) if err != nil { t.Fatal(err) } var won atomic.Int32 var wg sync.WaitGroup for k := 0; k < 8; k++ { wg.Add(1) go func() { defer wg.Done() if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok { won.Add(1) } }() } wg.Wait() if won.Load() != 1 { t.Fatalf("%d claims won", won.Load()) } if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again { t.Fatalf("a claim of a claimed wait: %v %v", again, err) } if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil { t.Fatalf("stuck %+v", rows) } if err := inv.ReleaseClaim(ctx, p.ID); err != nil { t.Fatal(err) } if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil { t.Fatalf("released %+v", rows) } // Only a wait that waits is withdrawn or expired, only a claim is applied. if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok { t.Fatal("a wait was applied") } if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok { t.Fatal("a wait that is not claimed was refused as a claim") } if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok { t.Fatalf("repoint %v %v", ok, err) } if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" { t.Fatalf("repointed %+v", rows) } } // **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one // making it to settle applied, and one still waiting is made by hand, and says so. func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) { inv, node := aNodeWithModules(t, "web", "cache") ctx := t.Context() claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}) if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok { t.Fatal(ok, err) } if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { t.Fatalf("assigned %v %v", added, err) } standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil || standing.Value.Pending.State != PendingApplying { t.Fatalf("the value that stands: %+v", standing.Value) } if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok { t.Fatalf("settled %v %v", ok, err) } if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" { t.Fatalf("assigned %v", on) } rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" { t.Fatalf("what became of the wait: %+v", rows) } waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"}) if _, err := inv.Assign(ctx, node, "cache"); err != nil { t.Fatal(err) } got, _ := inv.PendingFor(ctx, node, "cache") if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") { t.Fatalf("a wait assigned by hand: %+v", got) } } // **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094). func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) { inv, node := aNodeWithModules(t) ctx := t.Context() p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"}) if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil { t.Fatal(err) } if rows, _ := inv.ToRaise(ctx); len(rows) != 1 { t.Fatalf("to raise %+v", rows) } if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { t.Fatal(err) } first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) time.Sleep(2 * time.Millisecond) if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { t.Fatal(err) } second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) if !first[0].Raised.Equal(*second[0].Raised) { t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised) } if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil { t.Fatal("a mark nobody knows") } if rows, _ := inv.ToRaise(ctx); len(rows) != 0 { t.Fatalf("to raise %+v", rows) } if rows, _ := inv.ToClear(ctx); len(rows) != 1 { t.Fatalf("to clear %+v", rows) } if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { t.Fatal(err) } if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 { t.Fatalf("a wait outlived its machine: %+v", rows) } } // **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue // 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer // wait takes only the unanswered ones forward. func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) { inv, node := aNodeWithModules(t) ctx := t.Context() record := func(build string) PendingAssignment { p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build}) if err != nil { t.Fatal(err) } return p } end := func(p PendingAssignment) { if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok { t.Fatal(ok, err) } } first := record("b-1") end(first) if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil { t.Fatal(err) } second := record("b-2") if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] { t.Fatalf("known %v: the replaced wait's condition is unanswered", known) } if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID { t.Fatalf("to clear %+v: the first's condition is still to clear", clear) } rows, _ := inv.PendingFor(ctx, node, "sensors") if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID { t.Fatalf("the key says %+v, newest first", rows) } // The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it. end(second) if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID { t.Fatalf("to raise %+v", raise) } // A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered). if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil { t.Fatal(err) } third := record("b-3") if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID { t.Fatalf("the second was never raised and is to be: %+v", raise) } if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] { t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known) } }