mesh-host/internal/link/have_test.go

package link

import (
	"context"
	"encoding/json"
	"errors"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// What the node-engine writes beside its report: a module's have, and the node's own facts (novox/hq ADR 0429 §4,
// issue 1132).

// haveLink is a link with a bucket behind it that keeps each key's revision as the bus does.
type haveLink struct {
	*quietLink
	hmu                  sync.Mutex
	kv                   map[string]haveEntry
	sets                 []string // keys written, in order
	events               []string // "report" and "have <key>", in the order they happened
	failing              bool
	binds                int
	setTries             int
	bindFails, bindStuck bool
}

type haveEntry struct {
	value []byte
	rev   uint64
}

func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }

func (l *haveLink) Report(ctx context.Context, node string, body []byte) error {
	err := l.quietLink.Report(ctx, node, body)
	if err == nil {
		l.hmu.Lock()
		l.events = append(l.events, "report")
		l.hmu.Unlock()
	}
	return err
}

func (l *haveLink) Health(context.Context, string, []byte) error { return nil }

func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
	l.hmu.Lock()
	l.binds++
	stuck, refuse := l.bindStuck, l.bindFails
	l.hmu.Unlock()
	if stuck {
		<-ctx.Done()
		return nil, ctx.Err()
	}
	if refuse {
		return nil, errors.New("no grant for the have bucket")
	}
	return l, nil
}

func (l *haveLink) Get(_ context.Context, key string) ([]byte, uint64, error) {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	e, ok := l.kv[key]
	if !ok {
		return nil, 0, ErrHaveAbsent
	}
	return e.value, e.rev, nil
}

func (l *haveLink) Set(_ context.Context, key string, value []byte, revision uint64) error {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	l.setTries++
	if l.failing {
		return errors.New("the bucket refused")
	}
	e, ok := l.kv[key]
	if (revision == 0) == ok || (ok && e.rev != revision) {
		return errors.New("wrong last sequence")
	}
	l.kv[key] = haveEntry{value: value, rev: e.rev + 1}
	l.sets = append(l.sets, key)
	l.events = append(l.events, "have "+key)
	return nil
}

func (l *haveLink) have(t *testing.T, key string) delivery.Have {
	t.Helper()
	l.hmu.Lock()
	defer l.hmu.Unlock()
	h, err := delivery.DecodeHave(l.kv[key].value)
	if err != nil {
		t.Fatalf("%s holds no have: %v", key, err)
	}
	return h
}

func (l *haveLink) setCount() int {
	l.hmu.Lock()
	defer l.hmu.Unlock()
	return len(l.sets)
}

func (l *haveLink) put(t *testing.T, key string, h delivery.Have) {
	t.Helper()
	raw, err := delivery.EncodeHave(h)
	if err != nil {
		t.Fatal(err)
	}
	l.hmu.Lock()
	defer l.hmu.Unlock()
	l.kv[key] = haveEntry{value: raw, rev: l.kv[key].rev + 1}
}

func healthOf(states map[string]string) *Health {
	h := &Health{Contract: LivenessContract, At: time.Now().UTC()}
	for module, state := range states {
		h.Resources = append(h.Resources, ResourceHealth{Module: module, Resource: module + ".server", State: state})
	}
	return h
}

func appliedModules() Report {
	return Report{Declared: "d1", Applied: []string{"a.x", "b.x"},
		Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}},
		Health:  healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})}
}

func aHaveQueue(l *haveLink) (*Queue, *[]string) {
	var said []string
	var mu sync.Mutex
	q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) {
		mu.Lock()
		said = append(said, s)
		mu.Unlock()
	}}
	q.attach(context.Background(), l)
	return q, &said
}

// 1132/AC3, AC5: a module that applied is running with its build and a module that failed is failed with the detail;
// the key does not exist yet, so it is created.
func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	a, b := l.have(t, "n1.a"), l.have(t, "n1.b")
	if a.Phase != delivery.PhaseRunning || a.Build != "b-1" || a.Health != delivery.HealthOK {
		t.Errorf("a: %+v", a)
	}
	if b.Phase != delivery.PhaseFailed || b.Build != "b-2" || !strings.Contains(b.Detail, "no space") ||
		b.Health != delivery.HealthFailed {
		t.Errorf("b: %+v", b)
	}
	if l.kv["n1.a"].rev != 1 {
		t.Errorf("a was created at revision %d", l.kv["n1.a"].rev)
	}
}

// 1132/AC5: a have equal to the bucket's value is not written; a different one is written over the last revision.
func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) {
	l := newHaveLink()
	l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-0", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	q, _ := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	if l.kv["n1.a"].rev != 1 {
		t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev)
	}
	if l.kv["n1.b"].rev != 2 || l.have(t, "n1.b").Phase != delivery.PhaseFailed {
		t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev)
	}
	before := l.setCount()
	q.tell(context.Background(), appliedModules(), applied)
	if l.setCount() != before {
		t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before)
	}
}

// 1132/AC4: a look's health reaches the module's have: ok, failed, waits or unknown, and only a change is written.
func TestAModulesHealthFollowsTheLook(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	rep := Report{Declared: "d1", Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
		{Module: "c", Build: "b"}, {Module: "d", Build: "b"}}}
	q.tell(context.Background(), rep, applied)
	for _, m := range []string{"a", "b", "c", "d"} {
		if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown {
			t.Errorf("%s with no look is %s, want unknown", m, h)
		}
	}
	h := healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy, "c": StateStarting})
	h.Resources = append(h.Resources, ResourceHealth{Module: "a", Resource: "a.two", State: StateWaiting},
		ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown})
	if !q.SayHealth(context.Background(), *h) {
		t.Fatal("not said")
	}
	for m, want := range map[string]delivery.Health{"a": delivery.HealthWaits, "b": delivery.HealthFailed,
		"c": delivery.HealthWaits, "d": delivery.HealthUnknown} {
		if got := l.have(t, "n1."+m).Health; got != want {
			t.Errorf("%s is %s, want %s", m, got, want)
		}
	}
	before := l.setCount()
	h2 := *h
	q.SayHealth(context.Background(), h2)
	if l.setCount() != before {
		t.Errorf("the same look wrote %d keys", l.setCount()-before)
	}
	h2.Resources = []ResourceHealth{{Module: "a", Resource: "a.server", State: StateHealthy}}
	q.SayHealth(context.Background(), h2)
	if got := l.have(t, "n1.a").Health; got != delivery.HealthOK {
		t.Errorf("a after a healthy look is %s", got)
	}
}

// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters
// and profile.
func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	rep := appliedModules()
	rep.Windows = []Window{{Step: "backup", Holds: []string{"db"}}}
	rep.Busy = []Busy{{ID: "x", Container: "x", State: "waiting"}}
	rep.Strays = []Stray{{Kind: "container", Name: "old"}}
	rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
	rep.Profile = map[string]any{"cores": 8}
	q.tell(context.Background(), rep, applied)
	if len(l.events) < 2 || l.events[0] != "report" {
		t.Fatalf("the report was not first: %v", l.events)
	}
	raw, _ := json.Marshal(l.said()[0])
	if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") {
		t.Errorf("the report carries the module facts: %s", raw)
	}
	got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value)
	if err != nil {
		t.Fatal(err)
	}
	enc := func(v any) json.RawMessage { b, _ := json.Marshal(v); return b }
	want := delivery.HaveNode{V: delivery.Version, Windows: enc(rep.Windows), Busy: enc(rep.Busy), Strays: enc(rep.Strays),
		Filters: enc(rep.Filters), Profile: enc(rep.Profile)}
	if !delivery.SameHaveNode(got, want) {
		t.Errorf("the node key holds %+v, want %+v", got, want)
	}
}

// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
	l := newHaveLink()
	l.failing = true
	q, said := aHaveQueue(l)
	if !q.tell(context.Background(), appliedModules(), applied) {
		t.Fatal("the report did not go out")
	}
	if len(l.said()) != 1 {
		t.Fatalf("reports sent: %d", len(l.said()))
	}
	if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
		t.Errorf("the failure is not in the log: %v", *said)
	}
	l.hmu.Lock()
	l.failing = false
	l.hmu.Unlock()
	q.tell(context.Background(), appliedModules(), applied)
	if l.have(t, "n1.a").Phase != delivery.PhaseRunning {
		t.Error("the next apply did not write again")
	}
}

// A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong.
func TestNoBucketAndNoFactsWriteNothing(t *testing.T) {
	plain := newQuietLink()
	q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second}
	q.attach(context.Background(), plain)
	if !q.tell(context.Background(), appliedModules(), applied) {
		t.Fatal("the report did not go out")
	}
	l := newHaveLink()
	q2, _ := aHaveQueue(l)
	q2.tell(context.Background(), Report{Refused: "no"}, applied)
	if l.setCount() != 0 {
		t.Errorf("a refusal wrote %d keys", l.setCount())
	}
}

// 1132/AC5: a reconcile whose report is not news still writes the have keys that differ, so a node-engine that
// restarted writes them again; and writes nothing when they are the same.
func TestASilentReconcileStillWritesTheHaves(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.Reconcile = func(context.Context) (Report, bool) { return appliedModules(), true }
	q.reconcile(context.Background(), false)
	if len(l.said()) != 0 {
		t.Fatalf("a reconcile that is not news said %d reports", len(l.said()))
	}
	if l.have(t, "n1.a").Phase != delivery.PhaseRunning {
		t.Error("the reconcile wrote no have")
	}
	before := l.setCount()
	q.reconcile(context.Background(), false)
	if l.setCount() != before {
		t.Errorf("an unchanged reconcile wrote %d keys", l.setCount()-before)
	}
}

// 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that
// fails ends the pass, and is said once.
func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) {
	for name, set := range map[string]func(*haveLink){
		"a bind refused":            func(l *haveLink) { l.bindFails = true },
		"a bind that never answers": func(l *haveLink) { l.bindStuck = true },
	} {
		l := newHaveLink()
		set(l)
		q, said := aHaveQueue(l)
		q.Timeout = 100 * time.Millisecond
		start := time.Now()
		if !q.tell(context.Background(), appliedModules(), applied) {
			t.Fatalf("%s: the report did not go out", name)
		}
		if took := time.Since(start); took > time.Second {
			t.Errorf("%s: the pass took %s", name, took)
		}
		if l.binds != 1 || l.setTries != 0 || len(*said) != 1 {
			t.Errorf("%s: binds %d, writes tried %d, said %v; want one bind, no write, one line", name, l.binds, l.setTries, *said)
		}
	}
	l := newHaveLink()
	l.failing = true
	q, said := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	if l.binds != 1 || l.setTries != 1 || len(*said) != 1 {
		t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said)
	}
}

func onlyA() Report {
	return Report{Declared: "d2", Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}},
		Health: healthOf(map[string]string{"a": StateHealthy})}
}

// 1369/AC1: after a declaration with a and b, one with a only writes n1.b with the phase removed, over the revision
// read, and leaves a as it was; the key is not deleted.
func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev
	q.tell(context.Background(), onlyA(), applied)
	b := l.have(t, "n1.b")
	if b.Phase != delivery.PhaseRemoved || b.Build != "b-2" {
		t.Errorf("b: %+v", b)
	}
	if l.kv["n1.b"].rev != revB+1 {
		t.Errorf("b was written at revision %d, want one over %d", l.kv["n1.b"].rev, revB)
	}
	if l.kv["n1.a"].rev != revA || l.have(t, "n1.a").Phase != delivery.PhaseRunning {
		t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev)
	}
	before := l.setCount()
	q.tell(context.Background(), onlyA(), applied)
	if l.setCount() != before {
		t.Errorf("a second apply without b wrote %d keys, removed is written once", l.setCount()-before)
	}
}

// 1369/AC2: a module removed and later added back is written with its new phase.
func TestARemovedModuleAddedBackIsWrittenWithItsNewPhase(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	q.tell(context.Background(), onlyA(), applied)
	q.tell(context.Background(), appliedModules(), applied)
	if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseFailed || b.Build != "b-2" {
		t.Errorf("b after it came back: %+v", b)
	}
	q.tell(context.Background(), onlyA(), applied)
	if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
		t.Errorf("b removed again: %+v", b)
	}
}

// 1369/AC3: a module this node-engine never wrote a have key for is not written, and a key that is gone from the
// bucket is not created again.
func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) {
	l := newHaveLink()
	l.put(t, "n1.c", delivery.Have{V: delivery.Version, Build: "b-9", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	q, _ := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	q.tell(context.Background(), onlyA(), applied)
	if c := l.have(t, "n1.c"); c.Phase != delivery.PhaseRunning || l.kv["n1.c"].rev != 1 {
		t.Errorf("c, which this node-engine never wrote, changed: %+v at %d", c, l.kv["n1.c"].rev)
	}
	// a key that vanished from the bucket is not created by the removal
	l2 := newHaveLink()
	q2, _ := aHaveQueue(l2)
	q2.tell(context.Background(), appliedModules(), applied)
	l2.hmu.Lock()
	delete(l2.kv, "n1.b")
	l2.hmu.Unlock()
	q2.tell(context.Background(), onlyA(), applied)
	if _, there := l2.kv["n1.b"]; there {
		t.Error("the removal created a key that was not there")
	}
}

// A removal the bucket refused is written by the next pass.
func TestARefusedRemovalIsWrittenByTheNextPass(t *testing.T) {
	l := newHaveLink()
	q, _ := aHaveQueue(l)
	q.tell(context.Background(), appliedModules(), applied)
	l.hmu.Lock()
	l.failing = true
	l.hmu.Unlock()
	q.tell(context.Background(), onlyA(), applied)
	l.hmu.Lock()
	l.failing = false
	l.hmu.Unlock()
	q.tell(context.Background(), onlyA(), applied)
	if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
		t.Errorf("b: %+v", b)
	}
}

// 1369/F1: a node-engine that started since wrote nothing, and the report names the modules the kept declaration held
// before: one of them the new declaration lacks is written removed, one that is not in the bucket is not created.
func TestARestartedNodeEngineWritesTheRemovalFromTheReportsBefore(t *testing.T) {
	l := newHaveLink()
	l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-2", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
	q, _ := aHaveQueue(l) // fresh: nothing written by this process
	r := onlyA()
	r.Before = []string{"a", "b", "c"}
	q.tell(context.Background(), r, applied)
	if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
		t.Errorf("b: %+v", b)
	}
	if _, there := l.kv["n1.c"]; there {
		t.Error("c, which has no key, was created")
	}
	if l.kv["n1.a"].rev != 1 {
		t.Errorf("a was written, revision %d", l.kv["n1.a"].rev)
	}
}