package link import ( "context" "encoding/json" "errors" "strings" "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) // What the node-engine writes beside its report: a module's have, and the node's own facts (novox/hq ADR 0429 ยง4, // issue 1132). // haveLink is a link with a bucket behind it that keeps each key's revision as the bus does. type haveLink struct { *quietLink hmu sync.Mutex kv map[string]haveEntry sets []string // keys written, in order events []string // "report" and "have ", in the order they happened failing bool binds int setTries int bindFails, bindStuck bool } type haveEntry struct { value []byte rev uint64 } func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} } func (l *haveLink) Report(ctx context.Context, node string, body []byte) error { err := l.quietLink.Report(ctx, node, body) if err == nil { l.hmu.Lock() l.events = append(l.events, "report") l.hmu.Unlock() } return err } func (l *haveLink) Health(context.Context, string, []byte) error { return nil } func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) { l.hmu.Lock() l.binds++ stuck, refuse := l.bindStuck, l.bindFails l.hmu.Unlock() if stuck { <-ctx.Done() return nil, ctx.Err() } if refuse { return nil, errors.New("no grant for the have bucket") } return l, nil } func (l *haveLink) Get(_ context.Context, key string) ([]byte, uint64, error) { l.hmu.Lock() defer l.hmu.Unlock() e, ok := l.kv[key] if !ok { return nil, 0, ErrHaveAbsent } return e.value, e.rev, nil } func (l *haveLink) Set(_ context.Context, key string, value []byte, revision uint64) error { l.hmu.Lock() defer l.hmu.Unlock() l.setTries++ if l.failing { return errors.New("the bucket refused") } e, ok := l.kv[key] if (revision == 0) == ok || (ok && e.rev != revision) { return errors.New("wrong last sequence") } l.kv[key] = haveEntry{value: value, rev: e.rev + 1} l.sets = append(l.sets, key) l.events = append(l.events, "have "+key) return nil } func (l *haveLink) have(t *testing.T, key string) delivery.Have { t.Helper() l.hmu.Lock() defer l.hmu.Unlock() h, err := delivery.DecodeHave(l.kv[key].value) if err != nil { t.Fatalf("%s holds no have: %v", key, err) } return h } func (l *haveLink) setCount() int { l.hmu.Lock() defer l.hmu.Unlock() return len(l.sets) } func (l *haveLink) put(t *testing.T, key string, h delivery.Have) { t.Helper() raw, err := delivery.EncodeHave(h) if err != nil { t.Fatal(err) } l.hmu.Lock() defer l.hmu.Unlock() l.kv[key] = haveEntry{value: raw, rev: l.kv[key].rev + 1} } func healthOf(states map[string]string) *Health { h := &Health{Contract: LivenessContract, At: time.Now().UTC()} for module, state := range states { h.Resources = append(h.Resources, ResourceHealth{Module: module, Resource: module + ".server", State: state}) } return h } func appliedModules() Report { return Report{Declared: "d1", Applied: []string{"a.x", "b.x"}, Modules: []ModuleFact{{Module: "a", Build: "b-1"}, {Module: "b", Build: "b-2", Failed: true, Detail: "b.x failed: no space"}}, Health: healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy})} } func aHaveQueue(l *haveLink) (*Queue, *[]string) { var said []string var mu sync.Mutex q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second, Say: func(s string) { mu.Lock() said = append(said, s) mu.Unlock() }} q.attach(context.Background(), l) return q, &said } // 1132/AC3, AC5: a module that applied is running with its build and a module that failed is failed with the detail; // the key does not exist yet, so it is created. func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) q.tell(context.Background(), appliedModules(), applied) a, b := l.have(t, "n1.a"), l.have(t, "n1.b") if a.Phase != delivery.PhaseRunning || a.Build != "b-1" || a.Health != delivery.HealthOK { t.Errorf("a: %+v", a) } if b.Phase != delivery.PhaseFailed || b.Build != "b-2" || !strings.Contains(b.Detail, "no space") || b.Health != delivery.HealthFailed { t.Errorf("b: %+v", b) } if l.kv["n1.a"].rev != 1 { t.Errorf("a was created at revision %d", l.kv["n1.a"].rev) } } // 1132/AC5: a have equal to the bucket's value is not written; a different one is written over the last revision. func TestAHaveIsWrittenOnlyWhenItDiffers(t *testing.T) { l := newHaveLink() l.put(t, "n1.a", delivery.Have{V: delivery.Version, Build: "b-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) l.put(t, "n1.b", delivery.Have{V: delivery.Version, Build: "b-0", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) q, _ := aHaveQueue(l) q.tell(context.Background(), appliedModules(), applied) if l.kv["n1.a"].rev != 1 { t.Errorf("a equal to the bucket was written again, to revision %d", l.kv["n1.a"].rev) } if l.kv["n1.b"].rev != 2 || l.have(t, "n1.b").Phase != delivery.PhaseFailed { t.Errorf("b was not updated over its last revision: %+v at %d", l.have(t, "n1.b"), l.kv["n1.b"].rev) } before := l.setCount() q.tell(context.Background(), appliedModules(), applied) if l.setCount() != before { t.Errorf("an apply that changed nothing wrote %d keys", l.setCount()-before) } } // 1132/AC4: a look's health reaches the module's have: ok, failed, waits or unknown, and only a change is written. func TestAModulesHealthFollowsTheLook(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) rep := Report{Declared: "d1", Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"}, {Module: "c", Build: "b"}, {Module: "d", Build: "b"}}} q.tell(context.Background(), rep, applied) for _, m := range []string{"a", "b", "c", "d"} { if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown { t.Errorf("%s with no look is %s, want unknown", m, h) } } h := healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy, "c": StateStarting}) h.Resources = append(h.Resources, ResourceHealth{Module: "a", Resource: "a.two", State: StateWaiting}, ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown}) if !q.SayHealth(context.Background(), *h) { t.Fatal("not said") } for m, want := range map[string]delivery.Health{"a": delivery.HealthWaits, "b": delivery.HealthFailed, "c": delivery.HealthWaits, "d": delivery.HealthUnknown} { if got := l.have(t, "n1."+m).Health; got != want { t.Errorf("%s is %s, want %s", m, got, want) } } before := l.setCount() h2 := *h q.SayHealth(context.Background(), h2) if l.setCount() != before { t.Errorf("the same look wrote %d keys", l.setCount()-before) } h2.Resources = []ResourceHealth{{Module: "a", Resource: "a.server", State: StateHealthy}} q.SayHealth(context.Background(), h2) if got := l.have(t, "n1.a").Health; got != delivery.HealthOK { t.Errorf("a after a healthy look is %s", got) } } // 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters // and profile. func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) rep := appliedModules() rep.Windows = []Window{{Step: "backup", Holds: []string{"db"}}} rep.Busy = []Busy{{ID: "x", Container: "x", State: "waiting"}} rep.Strays = []Stray{{Kind: "container", Name: "old"}} rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}} rep.Profile = map[string]any{"cores": 8} q.tell(context.Background(), rep, applied) if len(l.events) < 2 || l.events[0] != "report" { t.Fatalf("the report was not first: %v", l.events) } raw, _ := json.Marshal(l.said()[0]) if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") { t.Errorf("the report carries the module facts: %s", raw) } got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value) if err != nil { t.Fatal(err) } enc := func(v any) json.RawMessage { b, _ := json.Marshal(v); return b } want := delivery.HaveNode{V: delivery.Version, Windows: enc(rep.Windows), Busy: enc(rep.Busy), Strays: enc(rep.Strays), Filters: enc(rep.Filters), Profile: enc(rep.Profile)} if !delivery.SameHaveNode(got, want) { t.Errorf("the node key holds %+v, want %+v", got, want) } } // 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again. func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) if !q.tell(context.Background(), appliedModules(), applied) { t.Fatal("the report did not go out") } if len(l.said()) != 1 { t.Fatalf("reports sent: %d", len(l.said())) } if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") { t.Errorf("the failure is not in the log: %v", *said) } l.hmu.Lock() l.failing = false l.hmu.Unlock() q.tell(context.Background(), appliedModules(), applied) if l.have(t, "n1.a").Phase != delivery.PhaseRunning { t.Error("the next apply did not write again") } } // A link without a bucket, and a report with no module facts (a refusal), write nothing and say nothing wrong. func TestNoBucketAndNoFactsWriteNothing(t *testing.T) { plain := newQuietLink() q := &Queue{Membership: Membership{Node: "n1"}, Timeout: time.Second} q.attach(context.Background(), plain) if !q.tell(context.Background(), appliedModules(), applied) { t.Fatal("the report did not go out") } l := newHaveLink() q2, _ := aHaveQueue(l) q2.tell(context.Background(), Report{Refused: "no"}, applied) if l.setCount() != 0 { t.Errorf("a refusal wrote %d keys", l.setCount()) } } // 1132/AC5: a reconcile whose report is not news still writes the have keys that differ, so a node-engine that // restarted writes them again; and writes nothing when they are the same. func TestASilentReconcileStillWritesTheHaves(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) q.Reconcile = func(context.Context) (Report, bool) { return appliedModules(), true } q.reconcile(context.Background(), false) if len(l.said()) != 0 { t.Fatalf("a reconcile that is not news said %d reports", len(l.said())) } if l.have(t, "n1.a").Phase != delivery.PhaseRunning { t.Error("the reconcile wrote no have") } before := l.setCount() q.reconcile(context.Background(), false) if l.setCount() != before { t.Errorf("an unchanged reconcile wrote %d keys", l.setCount()-before) } } // 1132/AC7: a bind that fails, or never answers, writes nothing and returns within the queue's timeout; a write that // fails ends the pass, and is said once. func TestAFailedBindOrWriteEndsThePassAndReturnsInTime(t *testing.T) { for name, set := range map[string]func(*haveLink){ "a bind refused": func(l *haveLink) { l.bindFails = true }, "a bind that never answers": func(l *haveLink) { l.bindStuck = true }, } { l := newHaveLink() set(l) q, said := aHaveQueue(l) q.Timeout = 100 * time.Millisecond start := time.Now() if !q.tell(context.Background(), appliedModules(), applied) { t.Fatalf("%s: the report did not go out", name) } if took := time.Since(start); took > time.Second { t.Errorf("%s: the pass took %s", name, took) } if l.binds != 1 || l.setTries != 0 || len(*said) != 1 { t.Errorf("%s: binds %d, writes tried %d, said %v; want one bind, no write, one line", name, l.binds, l.setTries, *said) } } l := newHaveLink() l.failing = true q, said := aHaveQueue(l) q.tell(context.Background(), appliedModules(), applied) if l.binds != 1 || l.setTries != 1 || len(*said) != 1 { t.Errorf("a failed write: binds %d, writes tried %d, said %v; want one of each and one line", l.binds, l.setTries, *said) } }