mesh-controller/internal/inventory/chain_pending_test.go

//go:build integration

package inventory

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/chain"
)

// **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state
// pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module.
func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) {
	inv, node := aNodeWithModules(t, "web")
	ctx := t.Context()
	p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog",
		Path: "/modules/sensors/"})
	if err != nil {
		t.Fatal(err)
	}
	if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" {
		t.Fatalf("recorded %+v, want id %s", p, want)
	}
	kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors")
	if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() ||
		kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" {
		t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err)
	}
	rows, err := inv.Pending(ctx, time.Time{})
	if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" {
		t.Fatalf("waiting: %+v %v", rows, err)
	}
	if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) {
		t.Fatalf("a second open wait: %v", err)
	}
	if _, err := inv.Assign(ctx, node, "web"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) {
		t.Fatalf("a wait for a module that is on the machine: %v", err)
	}
	// Written over once it ended: a settled wait is replaced by a new one under a new id.
	if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok {
		t.Fatalf("settled %v %v", ok, err)
	}
	next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"})
	if err != nil || next.ID == p.ID {
		t.Fatalf("a wait after one ended: %+v %v", next, err)
	}
	if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); known[p.ID] || !known[next.ID] {
		t.Fatalf("known %v: the key holds the newer one now", known)
	}
	if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 {
		t.Fatalf("the earlier wait is in the key's history: %d values", len(history))
	}
}

// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module
// runs nowhere, and a module only waiting for a build is not forgotten as assigned.
func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) {
	inv, node := aNodeWithModules(t, "web")
	ctx := t.Context()
	if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil {
		t.Fatal(err)
	}
	if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
		t.Fatalf("assigned %v", on)
	}
	if running, _ := inv.Running(ctx, "web"); len(running) != 0 {
		t.Fatalf("running %v", running)
	}
	if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 {
		t.Fatalf("runs %v", runs)
	}
	entries, _ := inv.Catalogued(ctx)
	for _, e := range entries {
		if e.Manifest.Module == "web" && len(e.On) != 0 {
			t.Fatalf("web is on %v", e.On)
		}
	}
	if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) {
		t.Fatalf("a module only waiting is assigned: %v", err)
	}
}

// **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision.
func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) {
	inv, node := aNodeWithModules(t)
	ctx := t.Context()
	p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
	if err != nil {
		t.Fatal(err)
	}
	var won atomic.Int32
	var wg sync.WaitGroup
	for k := 0; k < 8; k++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok {
				won.Add(1)
			}
		}()
	}
	wg.Wait()
	if won.Load() != 1 {
		t.Fatalf("%d claims won", won.Load())
	}
	if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again {
		t.Fatalf("a claim of a claimed wait: %v %v", again, err)
	}
	if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil {
		t.Fatalf("stuck %+v", rows)
	}
	if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
		t.Fatal(err)
	}
	if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil {
		t.Fatalf("released %+v", rows)
	}
	// Only a wait that waits is withdrawn or expired, only a claim is applied.
	if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok {
		t.Fatal("a wait was applied")
	}
	if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok {
		t.Fatal("a wait that is not claimed was refused as a claim")
	}
	if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok {
		t.Fatalf("repoint %v %v", ok, err)
	}
	if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" {
		t.Fatalf("repointed %+v", rows)
	}
}

// **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one
// making it to settle applied, and one still waiting is made by hand, and says so.
func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) {
	inv, node := aNodeWithModules(t, "web", "cache")
	ctx := t.Context()
	claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"})
	if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok {
		t.Fatal(ok, err)
	}
	if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
		t.Fatalf("assigned %v %v", added, err)
	}
	standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
	if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil ||
		standing.Value.Pending.State != PendingApplying {
		t.Fatalf("the value that stands: %+v", standing.Value)
	}
	if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok {
		t.Fatalf("settled %v %v", ok, err)
	}
	if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" {
		t.Fatalf("assigned %v", on)
	}
	rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
	if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" {
		t.Fatalf("what became of the wait: %+v", rows)
	}

	waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"})
	if _, err := inv.Assign(ctx, node, "cache"); err != nil {
		t.Fatal(err)
	}
	got, _ := inv.PendingFor(ctx, node, "cache")
	if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") {
		t.Fatalf("a wait assigned by hand: %+v", got)
	}
}

// **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094).
func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) {
	inv, node := aNodeWithModules(t)
	ctx := t.Context()
	p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
	if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil {
		t.Fatal(err)
	}
	if rows, _ := inv.ToRaise(ctx); len(rows) != 1 {
		t.Fatalf("to raise %+v", rows)
	}
	if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
		t.Fatal(err)
	}
	first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
	time.Sleep(2 * time.Millisecond)
	if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
		t.Fatal(err)
	}
	second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
	if !first[0].Raised.Equal(*second[0].Raised) {
		t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised)
	}
	if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil {
		t.Fatal("a mark nobody knows")
	}
	if rows, _ := inv.ToRaise(ctx); len(rows) != 0 {
		t.Fatalf("to raise %+v", rows)
	}
	if rows, _ := inv.ToClear(ctx); len(rows) != 1 {
		t.Fatalf("to clear %+v", rows)
	}
	if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
		t.Fatal(err)
	}
	if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 {
		t.Fatalf("a wait outlived its machine: %+v", rows)
	}
}

// **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue
// 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer
// wait takes only the unanswered ones forward.
func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) {
	inv, node := aNodeWithModules(t)
	ctx := t.Context()
	record := func(build string) PendingAssignment {
		p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build})
		if err != nil {
			t.Fatal(err)
		}
		return p
	}
	end := func(p PendingAssignment) {
		if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok {
			t.Fatal(ok, err)
		}
	}
	first := record("b-1")
	end(first)
	if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil {
		t.Fatal(err)
	}
	second := record("b-2")
	if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] {
		t.Fatalf("known %v: the replaced wait's condition is unanswered", known)
	}
	if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID {
		t.Fatalf("to clear %+v: the first's condition is still to clear", clear)
	}
	rows, _ := inv.PendingFor(ctx, node, "sensors")
	if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID {
		t.Fatalf("the key says %+v, newest first", rows)
	}
	// The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it.
	end(second)
	if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
		t.Fatalf("to raise %+v", raise)
	}
	// A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered).
	if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil {
		t.Fatal(err)
	}
	third := record("b-3")
	if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
		t.Fatalf("the second was never raised and is to be: %+v", raise)
	}
	if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] {
		t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known)
	}
}