192 lines
8.0 KiB
Go
192 lines
8.0 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
"github.com/novox/mesh-controller/internal/testbus"
|
|
)
|
|
|
|
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
|
|
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
|
|
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
|
|
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
|
|
if err := r.Retired(ctx, w); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
|
|
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
|
|
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
all, _ := conditionsFrom.Open(ctx)
|
|
if len(all) != 3 {
|
|
t.Fatalf("%+v", all)
|
|
}
|
|
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
left := openKeys(t, conditionsFrom)
|
|
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
|
|
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
|
|
}
|
|
}
|
|
|
|
func onATestBus(t *testing.T) *nats.Conn {
|
|
t.Helper()
|
|
url := testbus.URL(t)
|
|
js, err := broker.Dial(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(js.Close)
|
|
if err := js.EnsureControllerBuckets(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := handActConn
|
|
handActConn = js.Conn()
|
|
t.Cleanup(func() { handActConn = before })
|
|
return js.Conn()
|
|
}
|
|
|
|
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
|
|
conn := onATestBus(t)
|
|
fake := &fakeProvider{}
|
|
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
|
|
fake.serve(t, conn, "pg-approve", "anchor")
|
|
p := providerInstance{Node: "anchor", Module: "pg-approve"}
|
|
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
|
|
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
|
|
t.Fatalf("approved %v; said %s", fake.approved, said)
|
|
}
|
|
acts := handActsBy(t, conn, "retire approve")
|
|
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
|
|
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
|
|
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
|
|
}
|
|
|
|
// Reject, on another provider: the same set, and nothing approved.
|
|
other := &fakeProvider{state: fake.state}
|
|
other.serve(t, conn, "pg-reject", "anchor")
|
|
printed(t, func() error {
|
|
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
|
|
whyFlags(t, "still moving"))
|
|
})
|
|
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
|
|
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
|
|
}
|
|
|
|
// Nothing waiting: refused, nothing asked but the question.
|
|
idle := &fakeProvider{}
|
|
idle.serve(t, conn, "pg-idle", "anchor")
|
|
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
|
|
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
if len(idle.asked) != 1 {
|
|
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
|
|
}
|
|
}
|
|
|
|
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
|
|
conn := onATestBus(t)
|
|
fake := &fakeProvider{}
|
|
fake.state.Held = []string{"active"}
|
|
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
|
|
fake.serve(t, conn, "pg-clean", "anchor")
|
|
p := providerInstance{Node: "anchor", Module: "pg-clean"}
|
|
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
|
|
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
|
|
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
|
}
|
|
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
|
|
for _, name := range []string{"active", "nobody"} {
|
|
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
|
|
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
|
|
t.Fatalf("%s: %v", name, err)
|
|
}
|
|
}
|
|
if !slices.Equal(fake.deleted, []string{"old"}) {
|
|
t.Fatalf("more was deleted: %v", fake.deleted)
|
|
}
|
|
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
|
|
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
|
|
}
|
|
|
|
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
|
|
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
|
|
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
|
|
t.Fatalf("%+v", listing)
|
|
}
|
|
fake.deleted = nil
|
|
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
|
|
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
|
|
strings.Contains(said, "young") {
|
|
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
|
}
|
|
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
|
|
if !slices.Equal(fake.deleted, []string{"old"}) {
|
|
t.Fatalf("confirmed, deleted %v", fake.deleted)
|
|
}
|
|
}
|
|
|
|
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
|
|
conn := onATestBus(t)
|
|
old := &fakeProvider{}
|
|
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
|
|
old.serve(t, conn, "pg-d11-old", "anchor")
|
|
young := &fakeProvider{}
|
|
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
|
|
young.serve(t, conn, "pg-d11-young", "anchor")
|
|
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
|
|
// Nothing serves this one: a provider older than the question is passed over, not a failure.
|
|
{Node: "anchor", Module: "pg-d11-predates"}}
|
|
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
|
|
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
|
|
strings.Contains(found[0].Summary, "fresh") {
|
|
t.Fatalf("%+v", found)
|
|
}
|
|
}
|
|
|
|
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
|
|
conn := onATestBus(t)
|
|
fake := &fakeProvider{}
|
|
kept := retiredDaysAgo("ledger", 3)
|
|
kept.Access = "kept"
|
|
fake.state.Retired = []link.RetiredConsumer{kept}
|
|
fake.state.RetiresBy = "mark-only"
|
|
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
|
|
fake.serve(t, conn, "vault-mark", "anchor")
|
|
p := providerInstance{Node: "anchor", Module: "vault-mark"}
|
|
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
|
|
said := printed(t, func() error { return printRetired(listing, false) })
|
|
if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
|
|
t.Fatalf("%s %+v", said, listing)
|
|
}
|
|
said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
|
|
if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
|
|
t.Fatal(said)
|
|
}
|
|
}
|