mesh-tools
6a2fb63bfc
A module that left a node kept its have key at running for ever, so nothing could tell it stopped and the gate would never take the seat. The key is written once with the phase removed over the revision read, never deleted. Pins mesh-sdk to the commit that adds the phase.
430 lines
15 KiB
Go
430 lines
15 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// What the node-engine writes beside its report: a module's have, and the node's own facts (novox/hq ADR 0429 §4,
|
|
// issue 1132).
|
|
|
|
// haveLink is a link with a bucket behind it that keeps each key's revision as the bus does.
|
|
type haveLink struct {
|
|
*quietLink
|
|
hmu sync.Mutex
|
|
kv map[string]haveEntry
|
|
sets []string // keys written, in order
|
|
events []string // "report" and "have <key>", in the order they happened
|
|
failing bool
|
|
binds int
|
|
setTries int
|
|
bindFails, bindStuck bool
|
|
}
|
|
|
|
type haveEntry struct {
|
|
value []byte
|
|
rev uint64
|
|
}
|
|
|
|
func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }
|
|
|
|
func (l *haveLink) Report(ctx context.Context, node string, body []byte) error {
|
|
err := l.quietLink.Report(ctx, node, body)
|
|
if err == nil {
|
|
l.hmu.Lock()
|
|
l.events = append(l.events, "report")
|
|
l.hmu.Unlock()
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (l *haveLink) Health(context.Context, string, []byte) error { return nil }
|
|
|
|
func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
|
|
l.hmu.Lock()
|
|
l.binds++
|
|
stuck, refuse := l.bindStuck, l.bindFails
|
|
l.hmu.Unlock()
|
|
if stuck {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
if refuse {
|
|
return nil, errors.New("no grant for the have bucket")
|
|
}
|
|
return l, nil
|
|
}
|
|
|
|
func (l *haveLink) Get(_ context.Context, key string) ([]byte, uint64, error) {
|
|
l.hmu.Lock()
|
|
defer l.hmu.Unlock()
|
|
e, ok := l.kv[key]
|
|
if !ok {
|
|
return nil, 0, ErrHaveAbsent
|
|
}
|
|
return e.value, e.rev, nil
|
|
}
|
|
|
|
func (l *haveLink) Set(_ context.Context, key string, value []byte, revision uint64) error {
|
|
l.hmu.Lock()
|
|
defer l.hmu.Unlock()
|
|
l.setTries++
|
|
if l.failing {
|
|
return errors.New("the bucket refused")
|
|
}
|
|
e, ok := l.kv[key]
|
|
if (revision == 0) == ok || (ok && e.rev != revision) {
|
|
return errors.New("wrong last sequence")
|
|
}
|
|
l.kv[key] = haveEntry{value: value, rev: e.rev + 1}
|
|
l.sets = append(l.sets, key)
|
|
l.events = append(l.events, "have "+key)
|
|
return nil
|
|
}
|
|
|
|
func (l *haveLink) have(t *testing.T, key string) delivery.Have {
|
|
t.Helper()
|
|
l.hmu.Lock()
|
|
defer l.hmu.Unlock()
|
|
h, err := delivery.DecodeHave(l.kv[key].value)
|
|
if err != nil {
|
|
t.Fatalf("%s holds no have: %v", key, err)
|
|
}
|
|
return h
|
|
}
|
|
|
|
func (l *haveLink) setCount() int {
|
|
l.hmu.Lock()
|
|
defer l.hmu.Unlock()
|
|
return len(l.sets)
|
|
}
|
|
|
|
func (l *haveLink) put(t *testing.T, key string, h delivery.Have) {
|
|
t.Helper()
|
|
raw, err := delivery.EncodeHave(h)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
l.hmu.Lock()
|
|
defer l.hmu.Unlock()
|
|
l.kv[key] = haveEntry{value: raw, rev: l.kv[key].rev + 1}
|
|
}
|
|
|
|
func healthOf(states map[string]string) *Health {
|
|
h := &Health{Contract: LivenessContract, At: time.Now().UTC()}
|
|
for module, state := range states {
|
|
h.Resources = append(h.Resources, ResourceHealth{Module: module, Resource: module + ".server", State: state})
|
|
}
|
|
return h
|
|
}
|
|
|
|
func appliedModules() Report {
|
|
return Report{Declared: "d1", Applied: []string{"a.x", "b.x"},
|
|
Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}},
|
|
Health: healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})}
|
|
}
|
|
|
|
func aHaveQueue(l *haveLink) (*Queue, *[]string) {
|
|
var said []string
|
|
var mu sync.Mutex
|
|
q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) {
|
|
mu.Lock()
|
|
said = append(said, s)
|
|
mu.Unlock()
|
|
}}
|
|
q.attach(context.Background(), l)
|
|
return q, &said
|
|
}
|
|
|
|
// 1132/AC3, AC5: a module that applied is running with its build and a module that failed is failed with the detail;
|
|
// the key does not exist yet, so it is created.
|
|
func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
a, b := l.have(t, "n1.a"), l.have(t, "n1.b")
|
|
if a.Phase != delivery.PhaseRunning || a.Build != "b-1" || a.Health != delivery.HealthOK {
|
|
t.Errorf("a: %+v", a)
|
|
}
|
|
if b.Phase != delivery.PhaseFailed || b.Build != "b-2" || !strings.Contains(b.Detail, "no space") ||
|
|
b.Health != delivery.HealthFailed {
|
|
t.Errorf("b: %+v", b)
|
|
}
|
|
if l.kv["n1.a"].rev != 1 {
|
|
t.Errorf("a was created at revision %d", l.kv["n1.a"].rev)
|
|
}
|
|
}
|
|
|
|
// 1132/AC5: a have equal to the bucket's value is not written; a different one is written over the last revision.
|
|
func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) {
|
|
l := newHaveLink()
|
|
l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
|
l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-0", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
|
q, _ := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
if l.kv["n1.a"].rev != 1 {
|
|
t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev)
|
|
}
|
|
if l.kv["n1.b"].rev != 2 || l.have(t, "n1.b").Phase != delivery.PhaseFailed {
|
|
t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev)
|
|
}
|
|
before := l.setCount()
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
if l.setCount() != before {
|
|
t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before)
|
|
}
|
|
}
|
|
|
|
// 1132/AC4: a look's health reaches the module's have: ok, failed, waits or unknown, and only a change is written.
|
|
func TestAModulesHealthFollowsTheLook(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
rep := Report{Declared: "d1", Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
|
|
{Module: "c", Build: "b"}, {Module: "d", Build: "b"}}}
|
|
q.tell(context.Background(), rep, applied)
|
|
for _, m := range []string{"a", "b", "c", "d"} {
|
|
if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown {
|
|
t.Errorf("%s with no look is %s, want unknown", m, h)
|
|
}
|
|
}
|
|
h := healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy, "c": StateStarting})
|
|
h.Resources = append(h.Resources, ResourceHealth{Module: "a", Resource: "a.two", State: StateWaiting},
|
|
ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown})
|
|
if !q.SayHealth(context.Background(), *h) {
|
|
t.Fatal("not said")
|
|
}
|
|
for m, want := range map[string]delivery.Health{"a": delivery.HealthWaits, "b": delivery.HealthFailed,
|
|
"c": delivery.HealthWaits, "d": delivery.HealthUnknown} {
|
|
if got := l.have(t, "n1."+m).Health; got != want {
|
|
t.Errorf("%s is %s, want %s", m, got, want)
|
|
}
|
|
}
|
|
before := l.setCount()
|
|
h2 := *h
|
|
q.SayHealth(context.Background(), h2)
|
|
if l.setCount() != before {
|
|
t.Errorf("the same look wrote %d keys", l.setCount()-before)
|
|
}
|
|
h2.Resources = []ResourceHealth{{Module: "a", Resource: "a.server", State: StateHealthy}}
|
|
q.SayHealth(context.Background(), h2)
|
|
if got := l.have(t, "n1.a").Health; got != delivery.HealthOK {
|
|
t.Errorf("a after a healthy look is %s", got)
|
|
}
|
|
}
|
|
|
|
// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters
|
|
// and profile.
|
|
func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
rep := appliedModules()
|
|
rep.Windows = []Window{{Step: "backup", Holds: []string{"db"}}}
|
|
rep.Busy = []Busy{{ID: "x", Container: "x", State: "waiting"}}
|
|
rep.Strays = []Stray{{Kind: "container", Name: "old"}}
|
|
rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
|
|
rep.Profile = map[string]any{"cores": 8}
|
|
q.tell(context.Background(), rep, applied)
|
|
if len(l.events) < 2 || l.events[0] != "report" {
|
|
t.Fatalf("the report was not first: %v", l.events)
|
|
}
|
|
raw, _ := json.Marshal(l.said()[0])
|
|
if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") {
|
|
t.Errorf("the report carries the module facts: %s", raw)
|
|
}
|
|
got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
enc := func(v any) json.RawMessage { b, _ := json.Marshal(v); return b }
|
|
want := delivery.HaveNode{V: delivery.Version, Windows: enc(rep.Windows), Busy: enc(rep.Busy), Strays: enc(rep.Strays),
|
|
Filters: enc(rep.Filters), Profile: enc(rep.Profile)}
|
|
if !delivery.SameHaveNode(got, want) {
|
|
t.Errorf("the node key holds %+v, want %+v", got, want)
|
|
}
|
|
}
|
|
|
|
// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
|
|
func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
|
|
l := newHaveLink()
|
|
l.failing = true
|
|
q, said := aHaveQueue(l)
|
|
if !q.tell(context.Background(), appliedModules(), applied) {
|
|
t.Fatal("the report did not go out")
|
|
}
|
|
if len(l.said()) != 1 {
|
|
t.Fatalf("reports sent: %d", len(l.said()))
|
|
}
|
|
if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
|
|
t.Errorf("the failure is not in the log: %v", *said)
|
|
}
|
|
l.hmu.Lock()
|
|
l.failing = false
|
|
l.hmu.Unlock()
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
if l.have(t, "n1.a").Phase != delivery.PhaseRunning {
|
|
t.Error("the next apply did not write again")
|
|
}
|
|
}
|
|
|
|
// A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong.
|
|
func TestNoBucketAndNoFactsWriteNothing(t *testing.T) {
|
|
plain := newQuietLink()
|
|
q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second}
|
|
q.attach(context.Background(), plain)
|
|
if !q.tell(context.Background(), appliedModules(), applied) {
|
|
t.Fatal("the report did not go out")
|
|
}
|
|
l := newHaveLink()
|
|
q2, _ := aHaveQueue(l)
|
|
q2.tell(context.Background(), Report{Refused: "no"}, applied)
|
|
if l.setCount() != 0 {
|
|
t.Errorf("a refusal wrote %d keys", l.setCount())
|
|
}
|
|
}
|
|
|
|
// 1132/AC5: a reconcile whose report is not news still writes the have keys that differ, so a node-engine that
|
|
// restarted writes them again; and writes nothing when they are the same.
|
|
func TestASilentReconcileStillWritesTheHaves(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
q.Reconcile = func(context.Context) (Report, bool) { return appliedModules(), true }
|
|
q.reconcile(context.Background(), false)
|
|
if len(l.said()) != 0 {
|
|
t.Fatalf("a reconcile that is not news said %d reports", len(l.said()))
|
|
}
|
|
if l.have(t, "n1.a").Phase != delivery.PhaseRunning {
|
|
t.Error("the reconcile wrote no have")
|
|
}
|
|
before := l.setCount()
|
|
q.reconcile(context.Background(), false)
|
|
if l.setCount() != before {
|
|
t.Errorf("an unchanged reconcile wrote %d keys", l.setCount()-before)
|
|
}
|
|
}
|
|
|
|
// 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that
|
|
// fails ends the pass, and is said once.
|
|
func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) {
|
|
for name, set := range map[string]func(*haveLink){
|
|
"a bind refused": func(l *haveLink) { l.bindFails = true },
|
|
"a bind that never answers": func(l *haveLink) { l.bindStuck = true },
|
|
} {
|
|
l := newHaveLink()
|
|
set(l)
|
|
q, said := aHaveQueue(l)
|
|
q.Timeout = 100 * time.Millisecond
|
|
start := time.Now()
|
|
if !q.tell(context.Background(), appliedModules(), applied) {
|
|
t.Fatalf("%s: the report did not go out", name)
|
|
}
|
|
if took := time.Since(start); took > time.Second {
|
|
t.Errorf("%s: the pass took %s", name, took)
|
|
}
|
|
if l.binds != 1 || l.setTries != 0 || len(*said) != 1 {
|
|
t.Errorf("%s: binds %d, writes tried %d, said %v; want one bind, no write, one line", name, l.binds, l.setTries, *said)
|
|
}
|
|
}
|
|
l := newHaveLink()
|
|
l.failing = true
|
|
q, said := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
if l.binds != 1 || l.setTries != 1 || len(*said) != 1 {
|
|
t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said)
|
|
}
|
|
}
|
|
|
|
func onlyA() Report {
|
|
return Report{Declared: "d2", Applied: []string{"a.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}},
|
|
Health: healthOf(map[string]string{"a": StateHealthy})}
|
|
}
|
|
|
|
// 1369/AC1: after a declaration with a and b, one with a only writes n1.b with the phase removed, over the revision
|
|
// read, and leaves a as it was; the key is not deleted.
|
|
func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
b := l.have(t, "n1.b")
|
|
if b.Phase != delivery.PhaseRemoved || b.Build != "b-2" {
|
|
t.Errorf("b: %+v", b)
|
|
}
|
|
if l.kv["n1.b"].rev != revB+1 {
|
|
t.Errorf("b was written at revision %d, want one over %d", l.kv["n1.b"].rev, revB)
|
|
}
|
|
if l.kv["n1.a"].rev != revA || l.have(t, "n1.a").Phase != delivery.PhaseRunning {
|
|
t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev)
|
|
}
|
|
before := l.setCount()
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
if l.setCount() != before {
|
|
t.Errorf("a second apply without b wrote %d keys, removed is written once", l.setCount()-before)
|
|
}
|
|
}
|
|
|
|
// 1369/AC2: a module removed and later added back is written with its new phase.
|
|
func TestARemovedModuleAddedBackIsWrittenWithItsNewPhase(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseFailed || b.Build != "b-2" {
|
|
t.Errorf("b after it came back: %+v", b)
|
|
}
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
|
|
t.Errorf("b removed again: %+v", b)
|
|
}
|
|
}
|
|
|
|
// 1369/AC3: a module this node-engine never wrote a have key for is not written, and a key that is gone from the
|
|
// bucket is not created again.
|
|
func TestAModuleNeverWrittenIsNotWrittenRemoved(t *testing.T) {
|
|
l := newHaveLink()
|
|
l.put(t, "n1.c", delivery.Have{V: delivery.Version, Build: "b-9", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
|
q, _ := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
if c := l.have(t, "n1.c"); c.Phase != delivery.PhaseRunning || l.kv["n1.c"].rev != 1 {
|
|
t.Errorf("c, which this node-engine never wrote, changed: %+v at %d", c, l.kv["n1.c"].rev)
|
|
}
|
|
// a key that vanished from the bucket is not created by the removal
|
|
l2 := newHaveLink()
|
|
q2, _ := aHaveQueue(l2)
|
|
q2.tell(context.Background(), appliedModules(), applied)
|
|
l2.hmu.Lock()
|
|
delete(l2.kv, "n1.b")
|
|
l2.hmu.Unlock()
|
|
q2.tell(context.Background(), onlyA(), applied)
|
|
if _, there := l2.kv["n1.b"]; there {
|
|
t.Error("the removal created a key that was not there")
|
|
}
|
|
}
|
|
|
|
// A removal the bucket refused is written by the next pass.
|
|
func TestARefusedRemovalIsWrittenByTheNextPass(t *testing.T) {
|
|
l := newHaveLink()
|
|
q, _ := aHaveQueue(l)
|
|
q.tell(context.Background(), appliedModules(), applied)
|
|
l.hmu.Lock()
|
|
l.failing = true
|
|
l.hmu.Unlock()
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
l.hmu.Lock()
|
|
l.failing = false
|
|
l.hmu.Unlock()
|
|
q.tell(context.Background(), onlyA(), applied)
|
|
if b := l.have(t, "n1.b"); b.Phase != delivery.PhaseRemoved {
|
|
t.Errorf("b: %+v", b)
|
|
}
|
|
}
|