jschoubben
850f4b4d26
A refused key stays unacknowledged until its delay ends, so with 8 a consumer a handful refused for long (a manifest word this controller does not read yet, a node that keeps failing) stopped that consumer for good, the build that reads the word among what it held back. Each consumer now leaves up to 1000 unacknowledged; the store's bound is the one loop's batch of 16. A refusal waits 30 s, doubling each time up to 10 minutes. The stall watch judged movement by deliveries, which rise with every redelivery: it now counts the keys this process acknowledged. A consumer that cannot be made raises take-in-stalled instead of a log line. On stop the gathered batch and a key in hand are refused with no delay, so the next controller has them at once. A trunk value with no fingerprint of the new form is said once, as every wait for a write. Research 072: the take-ins on a bounded pull consumer.
300 lines
14 KiB
Go
300 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
|
)
|
|
|
|
// aTakeIn is a mesh for the take-in's steps: per module its registered commit, its trunk (commit, fingerprint), its
|
|
// nodes, whether its compositions and grant exist, and what its take-in answers; every act is written to acts.
|
|
type aTakeIn struct {
|
|
registered map[string]string
|
|
trunks map[string]string // module -> commit; no entry: no trunk value
|
|
nodes map[string][]string
|
|
taken map[string]bool
|
|
takeInErr map[string]error
|
|
composeErr map[string]error
|
|
grantsErr error
|
|
holder string
|
|
acts []string
|
|
}
|
|
|
|
func (m *aTakeIn) steps() takeInSteps {
|
|
fp := func(module string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(module[:1], 64) }
|
|
return takeInSteps{
|
|
catalogue: func(context.Context) (map[string][]string, error) {
|
|
m.acts = append(m.acts, "catalogue")
|
|
out := map[string][]string{}
|
|
for k, v := range m.nodes {
|
|
out[k] = v
|
|
}
|
|
return out, nil
|
|
},
|
|
trunk: func(_ context.Context, module string) (sdkdelivery.Module, error) {
|
|
commit, ok := m.trunks[module]
|
|
if !ok {
|
|
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
|
}
|
|
return sdkdelivery.Module{Commit: commit, FP2: fp(module)}, nil
|
|
},
|
|
registeredAt: func(_ context.Context, module, commit string) (bool, error) {
|
|
return m.registered[module] == commit, nil
|
|
},
|
|
takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error {
|
|
m.acts = append(m.acts, "take in "+module+" at "+trunk.Commit)
|
|
return m.takeInErr[module]
|
|
},
|
|
taken: func(_ context.Context, module, _ string, _ []string) (bool, error) { return m.taken[module], nil },
|
|
busHolder: func(context.Context) (string, error) { return m.holder, nil },
|
|
compose: func(_ context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
|
act := "compose " + strings.Join(nodes, ",")
|
|
if grants {
|
|
act += " and grants"
|
|
}
|
|
m.acts = append(m.acts, act)
|
|
failed := map[string]error{}
|
|
for _, n := range nodes {
|
|
if e := m.composeErr[n]; e != nil {
|
|
failed[n] = e
|
|
}
|
|
}
|
|
if grants {
|
|
return failed, m.grantsErr, nil
|
|
}
|
|
return failed, nil, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// A batch of done keys takes each module in once and asks the composer at most twice, whatever its size: once for the
|
|
// nodes of every module with something new, then once for the node holding the bus with the grants (ADR 0544 §5). A
|
|
// module registered at its trunk whose compositions and grant exist, or that this process took in at the same trunk,
|
|
// composes nothing (§3): the old take-in composed it three times over, every key and every start (issue 1945, 17:36 on
|
|
// 2026-10-25). A key delivered again is taken in whole. A wait for a write is acknowledged, a wait on a read and a
|
|
// failure refused, a failed build key acknowledged with its condition (§4).
|
|
func TestATakeInBatch(t *testing.T) {
|
|
waitsForAWrite := fmt.Errorf("%w: the key is building", errWaitsForAWrite)
|
|
waitsOnARead := fmt.Errorf("%w: the store is away", errKeyWaits)
|
|
keyFailed := fmt.Errorf("%w: the build failed", errKeyFailed)
|
|
type want struct {
|
|
verdict verdict
|
|
taken, nothingNew bool
|
|
waits, err bool
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
mesh aTakeIn
|
|
keys []delivered
|
|
memo map[string]string
|
|
acts []string
|
|
passes int
|
|
want map[string]want
|
|
}{
|
|
{name: "a new fingerprint: taken in, its nodes, then the bus's node and the grants",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "old"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace", "g14", "novox"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "take in mail at new", "compose ace,g14", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}}},
|
|
{name: "registered, composed and granted: nothing composed (fails on the old take-in)",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new", "nats": "n"}, trunks: map[string]string{"mail": "new", "nats": "n"},
|
|
nodes: map[string][]string{"mail": {"ace"}, "nats": {"novox"}}, taken: map[string]bool{"mail": true, "nats": true},
|
|
holder: "novox"},
|
|
keys: []delivered{{module: "mail"}, {module: "nats"}, {module: "mail"}},
|
|
acts: []string{"catalogue"},
|
|
want: map[string]want{"mail": {nothingNew: true}, "nats": {nothingNew: true}}},
|
|
{name: "taken in by this process at the same trunk: nothing new, not even a look at its keys",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
memo: map[string]string{"mail": sdkdelivery.SourcePrefix2 + strings.Repeat("m", 64) + "@new"},
|
|
acts: []string{"catalogue"}, want: map[string]want{"mail": {nothingNew: true}}},
|
|
{name: "delivered again: taken in whole, even when it looks done",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, taken: map[string]bool{"mail": true}, holder: "novox"},
|
|
keys: []delivered{{module: "mail", again: true}},
|
|
acts: []string{"catalogue", "compose ace", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}}},
|
|
{name: "a take-in that stopped half-way: composed",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "compose ace", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}}},
|
|
{name: "three modules, one pass for their nodes and one for the bus",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "web": "2", "bus": "3"},
|
|
nodes: map[string][]string{"mail": {"ace", "novox"}, "web": {"g14", "ace"}, "bus": {"novox"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}, {module: "web"}, {module: "bus"}},
|
|
acts: []string{"catalogue", "take in mail at 1", "take in web at 2", "take in bus at 3", "compose ace,g14",
|
|
"compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}, "web": {taken: true}, "bus": {taken: true}}},
|
|
{name: "assigned nowhere, held by no catalogue, no trunk value: nothing composed",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "odd": "2"},
|
|
nodes: map[string][]string{"mail": nil, "none": {"ace"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}, {module: "odd"}, {module: "none"}},
|
|
acts: []string{"catalogue", "take in mail at 1"},
|
|
want: map[string]want{"mail": {taken: true}, "odd": {}, "none": {}}},
|
|
{name: "a grant not written refuses the key, never swallowed",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox", grantsErr: errors.New("the bucket mesh-controller_grants: not found")},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "take in mail at 1", "compose ace", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {verdict: refused, err: true}}},
|
|
{name: "a node that fails refuses only its modules",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "web": "2"},
|
|
nodes: map[string][]string{"mail": {"ace"}, "web": {"g14"}}, holder: "novox",
|
|
composeErr: map[string]error{"ace": errors.New("store refused")}},
|
|
keys: []delivered{{module: "mail"}, {module: "web"}},
|
|
acts: []string{"catalogue", "take in mail at 1", "take in web at 2", "compose ace,g14", "compose novox and grants"},
|
|
passes: 2, want: map[string]want{"mail": {verdict: refused, err: true}, "web": {taken: true}}},
|
|
{name: "waits and failures: a write's wait acknowledged, a read's refused, a failed key acknowledged",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"a": "1", "b": "2", "c": "3"},
|
|
nodes: map[string][]string{"a": {"ace"}, "b": {"ace"}, "c": {"ace"}}, holder: "novox",
|
|
takeInErr: map[string]error{"a": waitsForAWrite, "b": waitsOnARead, "c": keyFailed}},
|
|
keys: []delivered{{module: "a"}, {module: "b"}, {module: "c"}},
|
|
acts: []string{"catalogue", "take in a at 1", "take in b at 2", "take in c at 3"},
|
|
want: map[string]want{"a": {waits: true}, "b": {verdict: refused, waits: true}, "c": {err: true}}},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
memo := c.memo
|
|
if memo == nil {
|
|
memo = map[string]string{}
|
|
}
|
|
settled, passes := c.mesh.steps().batch(t.Context(), c.keys, memo)
|
|
if !slices.Equal(c.mesh.acts, c.acts) {
|
|
t.Fatalf("the acts were\n %q\nwant\n %q", c.mesh.acts, c.acts)
|
|
}
|
|
if passes != c.passes {
|
|
t.Fatalf("%d composer pass(es), want %d", passes, c.passes)
|
|
}
|
|
if len(settled) != len(c.want) {
|
|
t.Fatalf("settled %d module(s), want %d: %+v", len(settled), len(c.want), settled)
|
|
}
|
|
for _, r := range settled {
|
|
w := c.want[r.module]
|
|
got := want{verdict: r.verdict, taken: r.taken, nothingNew: r.nothingNew, waits: r.waits != "", err: r.err != nil}
|
|
if got != w {
|
|
t.Errorf("%s: %+v, want %+v (%v)", r.module, got, w, r.err)
|
|
}
|
|
if r.taken && memo[r.module] == "" {
|
|
t.Errorf("%s was taken in and not remembered", r.module)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// And each delivered key is acknowledged or refused by its module's verdict, the refused with the retry's delay.
|
|
t.Run("acknowledged and refused", func(t *testing.T) {
|
|
mesh := aTakeIn{registered: map[string]string{}, trunks: map[string]string{"a": "1", "b": "2"},
|
|
nodes: map[string][]string{"a": {"ace"}, "b": {"ace"}}, takeInErr: map[string]error{"b": waitsOnARead}}
|
|
var acked, naked []string
|
|
msg := func(module string) takeInMsg {
|
|
return takeInMsg{delivered: delivered{module: module, deliveries: 3}, ack: func() { acked = append(acked, module) },
|
|
nak: func(d time.Duration) {
|
|
if d != refusalDelay(3) {
|
|
t.Errorf("refused with a delay of %s", d)
|
|
}
|
|
naked = append(naked, module)
|
|
}}
|
|
}
|
|
state := &takeInState{memo: map[string]string{}, raised: map[string]bool{}, saidWait: map[string]string{}}
|
|
said := state.take(t.Context(), mesh.steps(), []takeInMsg{msg("a"), msg("b"), msg("a")})
|
|
if !slices.Equal(acked, []string{"a", "a"}) || !slices.Equal(naked, []string{"b"}) {
|
|
t.Fatalf("acknowledged %v and refused %v", acked, naked)
|
|
}
|
|
if !strings.Contains(said, "1 taken in") || !strings.Contains(said, "1 waiting") {
|
|
t.Fatalf("said %q", said)
|
|
}
|
|
})
|
|
|
|
// A key refused for long holds no slot the others need: each consumer leaves far more keys unacknowledged than a
|
|
// batch takes, so 8 keys refused on each stop no ninth (the review of ADR 0544: with 8 a consumer, they stopped it
|
|
// for good), and a refusal waits longer each time, up to ten minutes.
|
|
t.Run("refused keys stop no other", func(t *testing.T) {
|
|
for _, bucket := range []string{sdkdelivery.ModulesBucket, sdkdelivery.BuilderBuildsBucket} {
|
|
if c := takeInConfig(bucket); c.MaxAckPending < 8+startTakeIns || c.AckPolicy != jetstream.AckExplicitPolicy {
|
|
t.Errorf("%s: %d unacknowledged at most (%v): 8 refused keys and a batch do not fit", bucket,
|
|
c.MaxAckPending, c.AckPolicy)
|
|
}
|
|
}
|
|
was := time.Duration(0)
|
|
for n := uint64(1); n <= 8; n++ {
|
|
d := refusalDelay(n)
|
|
if d < was || d > doneKeyRetryMost || (n == 1 && d != doneKeyRetry) {
|
|
t.Fatalf("the %d. refusal waits %s (the one before %s)", n, d, was)
|
|
}
|
|
was = d
|
|
}
|
|
if was != doneKeyRetryMost {
|
|
t.Fatalf("the 8. refusal waits %s, want %s", was, doneKeyRetryMost)
|
|
}
|
|
})
|
|
}
|
|
|
|
// A backlog that stands above zero with no key acknowledged for five minutes is a stall, however often its keys are
|
|
// delivered again; one that takes keys in, or empties, is not (ADR 0544 §6 and its review).
|
|
func TestATakeInStallsWhenItsBacklogStandsStill(t *testing.T) {
|
|
at := time.Date(2026, 10, 25, 17, 36, 0, 0, time.UTC)
|
|
w := &stallWatch{}
|
|
steps := []struct {
|
|
after time.Duration
|
|
backlog, acked uint64
|
|
stalled, changed bool
|
|
}{
|
|
{0, 190, 10, false, false},
|
|
{4 * time.Minute, 190, 10, false, false},
|
|
{5 * time.Minute, 190, 10, true, true}, // its keys delivered again all along, none acknowledged
|
|
{6 * time.Minute, 190, 10, true, false},
|
|
{7 * time.Minute, 180, 20, false, true},
|
|
{20 * time.Minute, 0, 20, false, false},
|
|
}
|
|
for i, s := range steps {
|
|
stalled, changed := w.look(takeInBacklog{Taking: s.backlog}, s.acked, at.Add(s.after))
|
|
if stalled != s.stalled || changed != s.changed {
|
|
t.Fatalf("reading %d: stalled %v changed %v, want %v %v", i, stalled, changed, s.stalled, s.changed)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1945/AC4: the bus's user list is a file of the bus module's composition (ADR 0539 clause 7): a change of the composed
|
|
// users is a new revision of that composition alone, every other composition of the node keeping its digest.
|
|
func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testing.T) {
|
|
body, fps := capturedBody(t, "g14")
|
|
fps["nats"] = sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64)
|
|
withUsers := func(users string) map[string]sdkdelivery.Composed {
|
|
var d map[string]any
|
|
if err := json.Unmarshal(body, &d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// As the composer writes it for the module holding mesh-broker (mesh-sdk catalogue: BusUsersID, owned by the module).
|
|
d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file",
|
|
"module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"})
|
|
raw, _ := json.Marshal(d)
|
|
comps, _, _, err := compositionsOf("g14", raw, fps, nil, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return comps
|
|
}
|
|
before, after := withUsers("authorization { users = [ {user: a} ] }\n"), withUsers("authorization { users = [ {user: a}, {user: b} ] }\n")
|
|
natsKey, _ := sdkdelivery.ComposedKey("g14", "nats", fps["nats"])
|
|
for key, c := range before {
|
|
d1, _ := sdkdelivery.ComposedDigestOf(c)
|
|
d2, _ := sdkdelivery.ComposedDigestOf(after[key])
|
|
if changed := d1 != d2; changed != (key == natsKey) {
|
|
t.Errorf("%s changed %v; only the bus module's composition changes with its user list", key, changed)
|
|
}
|
|
}
|
|
}
|