jschoubben
666d1b682e
mesh/delivery held for a person: merged without a passing check: only a person decides what follows
mesh/check checking 666d1b682e7f on 045f25bd1353
registeredAt compared commits, so a catalogue merge that moved only the commit (2df05030 at 18:02, a85d9037 before it) took every catalogue module in again and composed it: about 200 take-ins, some 25 minutes, on every mesh-catalog merge, which held nextcloud for an hour. A module whose trunk fingerprint is the one it is registered at now has the new commit recorded (built_from and source_head, as a registration moves them) and composes nothing unless its composition or grant is missing. Readers of the registered commit: the fingerprints read the commit's build or, with none, the indexer's value at it, which names the same fingerprint; status, modules, the board and the facts read it as the source's commit, which it is; the upgrade path counts it as built from that commit, which it is. The stall watch counted only settles the bus accepted, so each refused acknowledgement at a first start left work held for good: it now counts the loop's every attempt. The acknowledgement and refusals at the stop are sent past the stop, and the connection closes only after the readers and the loop have ended. Research 072: the take-ins on a bounded pull consumer.
339 lines
16 KiB
Go
339 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
|
)
|
|
|
|
// aTakeIn is a mesh for the take-in's steps: per module its registered commit, its trunk (commit, fingerprint), its
|
|
// nodes, whether its compositions and grant exist, and what its take-in answers; every act is written to acts.
|
|
type aTakeIn struct {
|
|
registered map[string]string
|
|
sameFP map[string]bool // registered at another commit, at the trunk's fingerprint
|
|
trunks map[string]string // module -> commit; no entry: no trunk value
|
|
nodes map[string][]string
|
|
taken map[string]bool
|
|
takeInErr map[string]error
|
|
composeErr map[string]error
|
|
grantsErr error
|
|
holder string
|
|
acts []string
|
|
}
|
|
|
|
func (m *aTakeIn) steps() takeInSteps {
|
|
fp := func(module string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(module[:1], 64) }
|
|
return takeInSteps{
|
|
catalogue: func(context.Context) (map[string][]string, error) {
|
|
m.acts = append(m.acts, "catalogue")
|
|
out := map[string][]string{}
|
|
for k, v := range m.nodes {
|
|
out[k] = v
|
|
}
|
|
return out, nil
|
|
},
|
|
trunk: func(_ context.Context, module string) (sdkdelivery.Module, error) {
|
|
commit, ok := m.trunks[module]
|
|
if !ok {
|
|
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
|
}
|
|
return sdkdelivery.Module{Commit: commit, FP2: fp(module)}, nil
|
|
},
|
|
registered: func(_ context.Context, module string, trunk sdkdelivery.Module) (bool, bool, error) {
|
|
if m.registered[module] == trunk.Commit {
|
|
return true, true, nil
|
|
}
|
|
return false, m.sameFP[module], nil
|
|
},
|
|
recordCommit: func(_ context.Context, module, commit string) error {
|
|
m.acts = append(m.acts, "record "+module+" at "+commit)
|
|
return nil
|
|
},
|
|
takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error {
|
|
m.acts = append(m.acts, "take in "+module+" at "+trunk.Commit)
|
|
return m.takeInErr[module]
|
|
},
|
|
taken: func(_ context.Context, module, _ string, _ []string) (bool, error) { return m.taken[module], nil },
|
|
busHolder: func(context.Context) (string, error) { return m.holder, nil },
|
|
compose: func(_ context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
|
act := "compose " + strings.Join(nodes, ",")
|
|
if grants {
|
|
act += " and grants"
|
|
}
|
|
m.acts = append(m.acts, act)
|
|
failed := map[string]error{}
|
|
for _, n := range nodes {
|
|
if e := m.composeErr[n]; e != nil {
|
|
failed[n] = e
|
|
}
|
|
}
|
|
if grants {
|
|
return failed, m.grantsErr, nil
|
|
}
|
|
return failed, nil, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// A batch of done keys takes each module in once and asks the composer at most twice, whatever its size: once for the
|
|
// nodes of every module with something new, then once for the node holding the bus with the grants (ADR 0544 §5). A
|
|
// module registered at its trunk whose compositions and grant exist, or that this process took in at the same trunk,
|
|
// composes nothing (§3): the old take-in composed it three times over, every key and every start (issue 1945, 17:36 on
|
|
// 2026-10-25). A key delivered again is taken in whole. A wait for a write is acknowledged, a wait on a read and a
|
|
// failure refused, a failed build key acknowledged with its condition (§4).
|
|
func TestATakeInBatch(t *testing.T) {
|
|
waitsForAWrite := fmt.Errorf("%w: the key is building", errWaitsForAWrite)
|
|
waitsOnARead := fmt.Errorf("%w: the store is away", errKeyWaits)
|
|
keyFailed := fmt.Errorf("%w: the build failed", errKeyFailed)
|
|
type want struct {
|
|
verdict verdict
|
|
taken, nothingNew bool
|
|
waits, err bool
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
mesh aTakeIn
|
|
keys []delivered
|
|
memo map[string]string
|
|
acts []string
|
|
passes int
|
|
want map[string]want
|
|
}{
|
|
{name: "a new fingerprint: taken in, its nodes, then the bus's node and the grants",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "old"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace", "g14", "novox"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "take in mail at new", "compose ace,g14", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}}},
|
|
{name: "registered, composed and granted: nothing composed (fails on the old take-in)",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new", "nats": "n"}, trunks: map[string]string{"mail": "new", "nats": "n"},
|
|
nodes: map[string][]string{"mail": {"ace"}, "nats": {"novox"}}, taken: map[string]bool{"mail": true, "nats": true},
|
|
holder: "novox"},
|
|
keys: []delivered{{module: "mail"}, {module: "nats"}, {module: "mail"}},
|
|
acts: []string{"catalogue"},
|
|
want: map[string]want{"mail": {nothingNew: true}, "nats": {nothingNew: true}}},
|
|
{name: "a catalogue merge that moved the commit and not the fingerprint: recorded, nothing composed",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "a85d9037"}, sameFP: map[string]bool{"mail": true},
|
|
trunks: map[string]string{"mail": "2df05030"}, nodes: map[string][]string{"mail": {"ace"}},
|
|
taken: map[string]bool{"mail": true}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "record mail at 2df05030"}, want: map[string]want{"mail": {nothingNew: true}}},
|
|
{name: "taken in by this process at the same trunk: nothing new, not even a look at its keys",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
memo: map[string]string{"mail": sdkdelivery.SourcePrefix2 + strings.Repeat("m", 64) + "@new"},
|
|
acts: []string{"catalogue"}, want: map[string]want{"mail": {nothingNew: true}}},
|
|
{name: "delivered again: taken in whole, even when it looks done",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, taken: map[string]bool{"mail": true}, holder: "novox"},
|
|
keys: []delivered{{module: "mail", again: true}},
|
|
acts: []string{"catalogue", "compose ace", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}}},
|
|
{name: "a take-in that stopped half-way: composed",
|
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "compose ace", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}}},
|
|
{name: "three modules, one pass for their nodes and one for the bus",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "web": "2", "bus": "3"},
|
|
nodes: map[string][]string{"mail": {"ace", "novox"}, "web": {"g14", "ace"}, "bus": {"novox"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}, {module: "web"}, {module: "bus"}},
|
|
acts: []string{"catalogue", "take in mail at 1", "take in web at 2", "take in bus at 3", "compose ace,g14",
|
|
"compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {taken: true}, "web": {taken: true}, "bus": {taken: true}}},
|
|
{name: "assigned nowhere, held by no catalogue, no trunk value: nothing composed",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "odd": "2"},
|
|
nodes: map[string][]string{"mail": nil, "none": {"ace"}}, holder: "novox"},
|
|
keys: []delivered{{module: "mail"}, {module: "odd"}, {module: "none"}},
|
|
acts: []string{"catalogue", "take in mail at 1"},
|
|
want: map[string]want{"mail": {taken: true}, "odd": {}, "none": {}}},
|
|
{name: "a grant not written refuses the key, never swallowed",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1"},
|
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox", grantsErr: errors.New("the bucket mesh-controller_grants: not found")},
|
|
keys: []delivered{{module: "mail"}},
|
|
acts: []string{"catalogue", "take in mail at 1", "compose ace", "compose novox and grants"}, passes: 2,
|
|
want: map[string]want{"mail": {verdict: refused, err: true}}},
|
|
{name: "a node that fails refuses only its modules",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "web": "2"},
|
|
nodes: map[string][]string{"mail": {"ace"}, "web": {"g14"}}, holder: "novox",
|
|
composeErr: map[string]error{"ace": errors.New("store refused")}},
|
|
keys: []delivered{{module: "mail"}, {module: "web"}},
|
|
acts: []string{"catalogue", "take in mail at 1", "take in web at 2", "compose ace,g14", "compose novox and grants"},
|
|
passes: 2, want: map[string]want{"mail": {verdict: refused, err: true}, "web": {taken: true}}},
|
|
{name: "waits and failures: a write's wait acknowledged, a read's refused, a failed key acknowledged",
|
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"a": "1", "b": "2", "c": "3"},
|
|
nodes: map[string][]string{"a": {"ace"}, "b": {"ace"}, "c": {"ace"}}, holder: "novox",
|
|
takeInErr: map[string]error{"a": waitsForAWrite, "b": waitsOnARead, "c": keyFailed}},
|
|
keys: []delivered{{module: "a"}, {module: "b"}, {module: "c"}},
|
|
acts: []string{"catalogue", "take in a at 1", "take in b at 2", "take in c at 3"},
|
|
want: map[string]want{"a": {waits: true}, "b": {verdict: refused, waits: true}, "c": {err: true}}},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
memo := c.memo
|
|
if memo == nil {
|
|
memo = map[string]string{}
|
|
}
|
|
settled, passes := c.mesh.steps().batch(t.Context(), c.keys, memo)
|
|
if !slices.Equal(c.mesh.acts, c.acts) {
|
|
t.Fatalf("the acts were\n %q\nwant\n %q", c.mesh.acts, c.acts)
|
|
}
|
|
if passes != c.passes {
|
|
t.Fatalf("%d composer pass(es), want %d", passes, c.passes)
|
|
}
|
|
if len(settled) != len(c.want) {
|
|
t.Fatalf("settled %d module(s), want %d: %+v", len(settled), len(c.want), settled)
|
|
}
|
|
for _, r := range settled {
|
|
w := c.want[r.module]
|
|
got := want{verdict: r.verdict, taken: r.taken, nothingNew: r.nothingNew, waits: r.waits != "", err: r.err != nil}
|
|
if got != w {
|
|
t.Errorf("%s: %+v, want %+v (%v)", r.module, got, w, r.err)
|
|
}
|
|
if r.taken && memo[r.module] == "" {
|
|
t.Errorf("%s was taken in and not remembered", r.module)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// And each delivered key is acknowledged or refused by its module's verdict, the refused with the retry's delay.
|
|
t.Run("acknowledged and refused", func(t *testing.T) {
|
|
mesh := aTakeIn{registered: map[string]string{}, trunks: map[string]string{"a": "1", "b": "2"},
|
|
nodes: map[string][]string{"a": {"ace"}, "b": {"ace"}}, takeInErr: map[string]error{"b": waitsOnARead}}
|
|
var acked, naked []string
|
|
msg := func(module string) takeInMsg {
|
|
return takeInMsg{delivered: delivered{module: module, deliveries: 3}, ack: func() { acked = append(acked, module) },
|
|
nak: func(d time.Duration) {
|
|
if d != refusalDelay(3) {
|
|
t.Errorf("refused with a delay of %s", d)
|
|
}
|
|
naked = append(naked, module)
|
|
}}
|
|
}
|
|
state := &takeInState{memo: map[string]string{}, raised: map[string]bool{}, saidWait: map[string]string{}}
|
|
said := state.take(t.Context(), mesh.steps(), []takeInMsg{msg("a"), msg("b"), msg("a")})
|
|
if !slices.Equal(acked, []string{"a", "a"}) || !slices.Equal(naked, []string{"b"}) {
|
|
t.Fatalf("acknowledged %v and refused %v", acked, naked)
|
|
}
|
|
if !strings.Contains(said, "1 taken in") || !strings.Contains(said, "1 waiting") {
|
|
t.Fatalf("said %q", said)
|
|
}
|
|
})
|
|
|
|
// A key refused for long holds no slot the others need: each consumer leaves far more keys unacknowledged than a
|
|
// batch takes, so 8 keys refused on each stop no ninth (the review of ADR 0544: with 8 a consumer, they stopped it
|
|
// for good), and a refusal waits longer each time, up to ten minutes.
|
|
t.Run("refused keys stop no other", func(t *testing.T) {
|
|
for _, bucket := range []string{sdkdelivery.ModulesBucket, sdkdelivery.BuilderBuildsBucket} {
|
|
if c := takeInConfig(bucket); c.MaxAckPending < 8+startTakeIns || c.AckPolicy != jetstream.AckExplicitPolicy {
|
|
t.Errorf("%s: %d unacknowledged at most (%v): 8 refused keys and a batch do not fit", bucket,
|
|
c.MaxAckPending, c.AckPolicy)
|
|
}
|
|
}
|
|
was := time.Duration(0)
|
|
for n := uint64(1); n <= 8; n++ {
|
|
d := refusalDelay(n)
|
|
if d < was || d > doneKeyRetryMost || (n == 1 && d != doneKeyRetry) {
|
|
t.Fatalf("the %d. refusal waits %s (the one before %s)", n, d, was)
|
|
}
|
|
was = d
|
|
}
|
|
if was != doneKeyRetryMost {
|
|
t.Fatalf("the 8. refusal waits %s, want %s", was, doneKeyRetryMost)
|
|
}
|
|
})
|
|
}
|
|
|
|
// The take-in stalls when it holds work (keys not delivered yet, or a batch handed and not settled) and settles none
|
|
// for five minutes; refused keys waiting for their delay are no work, so a quiet mesh is no stall. The first look
|
|
// always says what it found, so a stall raised before a restart is cleared once this controller moves (ADR 0544 §6
|
|
// and its second review).
|
|
func TestATakeInStallsWhenItHoldsWorkAndSettlesNone(t *testing.T) {
|
|
at := time.Date(2026, 10, 25, 17, 36, 0, 0, time.UTC)
|
|
type reading struct {
|
|
after time.Duration
|
|
pending, taking uint64
|
|
handed, settled uint64
|
|
stalled, changed bool
|
|
}
|
|
for _, c := range []struct {
|
|
name string
|
|
readings []reading
|
|
}{
|
|
{"keys not delivered and nothing settled", []reading{
|
|
{0, 190, 8, 50, 50, false, true}, // the first look always says what it found
|
|
{4 * time.Minute, 190, 8, 50, 50, false, false},
|
|
{5 * time.Minute, 190, 8, 50, 50, true, true},
|
|
{6 * time.Minute, 190, 8, 50, 50, true, false},
|
|
{7 * time.Minute, 180, 8, 60, 60, false, true},
|
|
{20 * time.Minute, 0, 0, 60, 60, false, false},
|
|
}},
|
|
{"refused keys waiting alone are no stall", []reading{
|
|
{0, 0, 3, 40, 40, false, true},
|
|
{30 * time.Minute, 0, 3, 40, 40, false, false},
|
|
}},
|
|
{"every key settled, one of them refused by the bus, nothing pending: no stall", []reading{
|
|
{0, 0, 1, 3, 3, false, true},
|
|
{6 * time.Minute, 0, 1, 3, 3, false, false},
|
|
}},
|
|
{"a batch handed and never settled is a stall", []reading{
|
|
{0, 0, 16, 56, 40, false, true},
|
|
{5 * time.Minute, 0, 16, 56, 40, true, true},
|
|
}},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
w := &stallWatch{}
|
|
for i, r := range c.readings {
|
|
stalled, changed := w.look(takeInBacklog{Pending: r.pending, Taking: r.taking}, r.handed, r.settled,
|
|
at.Add(r.after))
|
|
if stalled != r.stalled || changed != r.changed {
|
|
t.Fatalf("reading %d: stalled %v changed %v, want %v %v", i, stalled, changed, r.stalled, r.changed)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// 1945/AC4: the bus's user list is a file of the bus module's composition (ADR 0539 clause 7): a change of the composed
|
|
// users is a new revision of that composition alone, every other composition of the node keeping its digest.
|
|
func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testing.T) {
|
|
body, fps := capturedBody(t, "g14")
|
|
fps["nats"] = sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64)
|
|
withUsers := func(users string) map[string]sdkdelivery.Composed {
|
|
var d map[string]any
|
|
if err := json.Unmarshal(body, &d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// As the composer writes it for the module holding mesh-broker (mesh-sdk catalogue: BusUsersID, owned by the module).
|
|
d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file",
|
|
"module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"})
|
|
raw, _ := json.Marshal(d)
|
|
comps, _, _, err := compositionsOf("g14", raw, fps, nil, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return comps
|
|
}
|
|
before, after := withUsers("authorization { users = [ {user: a} ] }\n"), withUsers("authorization { users = [ {user: a}, {user: b} ] }\n")
|
|
natsKey, _ := sdkdelivery.ComposedKey("g14", "nats", fps["nats"])
|
|
for key, c := range before {
|
|
d1, _ := sdkdelivery.ComposedDigestOf(c)
|
|
d2, _ := sdkdelivery.ComposedDigestOf(after[key])
|
|
if changed := d1 != d2; changed != (key == natsKey) {
|
|
t.Errorf("%s changed %v; only the bus module's composition changes with its user list", key, changed)
|
|
}
|
|
}
|
|
}
|