mesh-controller/cmd/mesh-controller/found_wait_test.go

package main

import (
	"context"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
	return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
		Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
		Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
			"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
}

func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
	now := time.Now()
	sent := now.Add(-time.Minute)
	f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
	h, why := moduleHealthWord("notes", "laptop", sent, f)
	if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
		t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
	}
	// Found by this very send: the send brought it, and it is not passed.
	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
		t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
	}
	// A container down beside the old wait is a fault, as before.
	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
		foundDirectory("notes", sent.Add(-time.Hour)),
		{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
		t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
	}
}

// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
	for _, c := range []struct {
		name   string
		found  time.Duration // when the directory was found, against now
		passes bool
	}{
		{"found a day before the send", -24 * time.Hour, true},
		{"found by this send", time.Hour, false},
	} {
		t.Run(c.name, func(t *testing.T) {
			b := aBacklog(t)
			ctx := t.Context()
			inv := b.open.inventory
			releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
				return map[string]bool{"anchor": true, "laptop": true}, nil
			}
			backlogFacts := gatherGateFacts
			gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
				f, err := backlogFacts(ctx, open, component)
				// The used-as-found condition, raised after the send (its second statement): its own kind, never a
				// fault the gate reads as the build's.
				f.judged, f.openErr = true, nil
				f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
					Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
					Raised:  time.Now()})
				f.health = map[string]inventory.NodeHealth{}
				for _, n := range []string{"anchor", "laptop"} {
					f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
						{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
						foundDirectory("app", time.Now().Add(c.found)),
						{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
				}
				return f, err
			}
			wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
			t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
			gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
			deadline := time.Now().Add(5 * time.Second)
			for time.Now().Before(deadline) {
				advancePlans(ctx, b.open)
				if p := b.release(t); p.State != inventory.PlanRolling {
					break
				}
				time.Sleep(20 * time.Millisecond)
			}
			p := b.release(t)
			v, found, err := inv.GateOf(ctx, "build-app-c2")
			if c.passes {
				if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
					t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
				}
				if !strings.Contains(v.Why+p.Note, "hand it over") {
					t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
				}
				return
			}
			if p.State == inventory.PlanDone {
				t.Fatalf("a directory this send found let the walk through: %s", p.Note)
			}
		})
	}
}

// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	ctx := t.Context()
	rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
	for i := 0; i < 2; i++ {
		if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
			t.Fatal(err)
		}
	}
	open, _ := k.Open(ctx)
	var got *conditions.Condition
	for i, c := range open {
		if c.Key == usedAsFoundKey("notes", "laptop") {
			got = &open[i]
		}
		if c.Kind == kindModuleUnhealthy {
			t.Fatalf("raised as a fault: %+v", c)
		}
	}
	if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
		!strings.Contains(got.Summary, "notes.data") {
		t.Fatalf("the condition: %+v", got)
	}
	// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
	if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
		t.Fatal(err)
	}
	open, _ = k.Open(ctx)
	for _, c := range open {
		if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
			t.Fatal("a directory used as found became urgent")
		}
	}
	if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
		t.Fatal(err)
	}
	open, _ = k.Open(ctx)
	for _, c := range open {
		if c.Key == usedAsFoundKey("notes", "laptop") {
			t.Fatal("not cleared once handed over")
		}
	}
}