jschoubben
3b989d8bb3
A wait that ended unmade and whose condition nobody answered is carried by the wait that replaces it, so the operator is never left without the condition for a module still not on its machine, and taking it back, its condition and the orphan sweep read it there. The condition id stays node.module.<id> with the string id. clearOrphaned says a taken-back wait too. TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket enters through the assign verb and reads the key before and after the build's outcome.
274 lines
11 KiB
Go
274 lines
11 KiB
Go
//go:build integration
|
|
|
|
package inventory
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/chain"
|
|
)
|
|
|
|
// **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state
|
|
// pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module.
|
|
func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) {
|
|
inv, node := aNodeWithModules(t, "web")
|
|
ctx := t.Context()
|
|
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog",
|
|
Path: "/modules/sensors/"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" {
|
|
t.Fatalf("recorded %+v, want id %s", p, want)
|
|
}
|
|
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors")
|
|
if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() ||
|
|
kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" {
|
|
t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err)
|
|
}
|
|
rows, err := inv.Pending(ctx, time.Time{})
|
|
if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" {
|
|
t.Fatalf("waiting: %+v %v", rows, err)
|
|
}
|
|
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) {
|
|
t.Fatalf("a second open wait: %v", err)
|
|
}
|
|
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) {
|
|
t.Fatalf("a wait for a module that is on the machine: %v", err)
|
|
}
|
|
// Written over once it ended: a settled wait is replaced by a new one under a new id.
|
|
if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok {
|
|
t.Fatalf("settled %v %v", ok, err)
|
|
}
|
|
next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"})
|
|
if err != nil || next.ID == p.ID {
|
|
t.Fatalf("a wait after one ended: %+v %v", next, err)
|
|
}
|
|
if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); known[p.ID] || !known[next.ID] {
|
|
t.Fatalf("known %v: the key holds the newer one now", known)
|
|
}
|
|
if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 {
|
|
t.Fatalf("the earlier wait is in the key's history: %d values", len(history))
|
|
}
|
|
if n := rowsInTheTable(t, inv, "pending_assignment"); n != "0" {
|
|
t.Fatalf("the table took %s pending assignments", n)
|
|
}
|
|
}
|
|
|
|
// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module
|
|
// runs nowhere, and a module only waiting for a build is not forgotten as assigned.
|
|
func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) {
|
|
inv, node := aNodeWithModules(t, "web")
|
|
ctx := t.Context()
|
|
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
|
|
t.Fatalf("assigned %v", on)
|
|
}
|
|
if running, _ := inv.Running(ctx, "web"); len(running) != 0 {
|
|
t.Fatalf("running %v", running)
|
|
}
|
|
if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 {
|
|
t.Fatalf("runs %v", runs)
|
|
}
|
|
entries, _ := inv.Catalogued(ctx)
|
|
for _, e := range entries {
|
|
if e.Manifest.Module == "web" && len(e.On) != 0 {
|
|
t.Fatalf("web is on %v", e.On)
|
|
}
|
|
}
|
|
if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) {
|
|
t.Fatalf("a module only waiting is assigned: %v", err)
|
|
}
|
|
}
|
|
|
|
// **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision.
|
|
func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var won atomic.Int32
|
|
var wg sync.WaitGroup
|
|
for k := 0; k < 8; k++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok {
|
|
won.Add(1)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if won.Load() != 1 {
|
|
t.Fatalf("%d claims won", won.Load())
|
|
}
|
|
if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again {
|
|
t.Fatalf("a claim of a claimed wait: %v %v", again, err)
|
|
}
|
|
if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil {
|
|
t.Fatalf("stuck %+v", rows)
|
|
}
|
|
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil {
|
|
t.Fatalf("released %+v", rows)
|
|
}
|
|
// Only a wait that waits is withdrawn or expired, only a claim is applied.
|
|
if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok {
|
|
t.Fatal("a wait was applied")
|
|
}
|
|
if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok {
|
|
t.Fatal("a wait that is not claimed was refused as a claim")
|
|
}
|
|
if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok {
|
|
t.Fatalf("repoint %v %v", ok, err)
|
|
}
|
|
if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" {
|
|
t.Fatalf("repointed %+v", rows)
|
|
}
|
|
}
|
|
|
|
// **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one
|
|
// making it to settle applied, and one still waiting is made by hand, and says so.
|
|
func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) {
|
|
inv, node := aNodeWithModules(t, "web", "cache")
|
|
ctx := t.Context()
|
|
claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"})
|
|
if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
|
|
t.Fatalf("assigned %v %v", added, err)
|
|
}
|
|
standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
|
|
if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil ||
|
|
standing.Value.Pending.State != PendingApplying {
|
|
t.Fatalf("the value that stands: %+v", standing.Value)
|
|
}
|
|
if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok {
|
|
t.Fatalf("settled %v %v", ok, err)
|
|
}
|
|
if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" {
|
|
t.Fatalf("assigned %v", on)
|
|
}
|
|
rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
|
|
if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" {
|
|
t.Fatalf("what became of the wait: %+v", rows)
|
|
}
|
|
|
|
waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"})
|
|
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ := inv.PendingFor(ctx, node, "cache")
|
|
if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") {
|
|
t.Fatalf("a wait assigned by hand: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094).
|
|
func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
|
|
if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rows, _ := inv.ToRaise(ctx); len(rows) != 1 {
|
|
t.Fatalf("to raise %+v", rows)
|
|
}
|
|
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
|
|
time.Sleep(2 * time.Millisecond)
|
|
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
|
|
if !first[0].Raised.Equal(*second[0].Raised) {
|
|
t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised)
|
|
}
|
|
if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil {
|
|
t.Fatal("a mark nobody knows")
|
|
}
|
|
if rows, _ := inv.ToRaise(ctx); len(rows) != 0 {
|
|
t.Fatalf("to raise %+v", rows)
|
|
}
|
|
if rows, _ := inv.ToClear(ctx); len(rows) != 1 {
|
|
t.Fatalf("to clear %+v", rows)
|
|
}
|
|
if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 {
|
|
t.Fatalf("a wait outlived its machine: %+v", rows)
|
|
}
|
|
}
|
|
|
|
// **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue
|
|
// 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer
|
|
// wait takes only the unanswered ones forward.
|
|
func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) {
|
|
inv, node := aNodeWithModules(t)
|
|
ctx := t.Context()
|
|
record := func(build string) PendingAssignment {
|
|
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
end := func(p PendingAssignment) {
|
|
if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok {
|
|
t.Fatal(ok, err)
|
|
}
|
|
}
|
|
first := record("b-1")
|
|
end(first)
|
|
if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second := record("b-2")
|
|
if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] {
|
|
t.Fatalf("known %v: the replaced wait's condition is unanswered", known)
|
|
}
|
|
if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID {
|
|
t.Fatalf("to clear %+v: the first's condition is still to clear", clear)
|
|
}
|
|
rows, _ := inv.PendingFor(ctx, node, "sensors")
|
|
if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID {
|
|
t.Fatalf("the key says %+v, newest first", rows)
|
|
}
|
|
// The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it.
|
|
end(second)
|
|
if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
|
|
t.Fatalf("to raise %+v", raise)
|
|
}
|
|
// A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered).
|
|
if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
third := record("b-3")
|
|
if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
|
|
t.Fatalf("the second was never raised and is to be: %+v", raise)
|
|
}
|
|
if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] {
|
|
t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known)
|
|
}
|
|
}
|