assigner: test 108, the composition runs started whole against the bus

One test of the whole assigner: one run per candidate and machine read by every step, a
module's fault failing only its own steps, the bound of owed progress, the untaken ask, a
step-ended after the bound ignored, a rerun moving the reader that did not pass, a
superseded step leaving its run, a restart taking a run up by its ask, the drop verb and
the cap.

Builds: design 47 G §733 (test 108); A-470f
This commit is contained in:
2026-11-17 12:07:56 +01:00
parent 930a407a3a
commit 1878607208
2 changed files with 392 additions and 5 deletions
@@ -0,0 +1,378 @@
//go:build integration
package main
import (
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-assigner/internal/buckets"
)
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
// taken ends at run-untaken-bound. A rerun of a starts one run on the machine it did not pass and moves b; a superseded step leaves
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
b := startBed(t, "")
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const commit = "1111111111111111111111111111111111111111"
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
for _, m := range []string{"a", "b", "c"} {
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2"} {
for _, m := range []string{"a", "b", "c"} {
b.place(t, node, m)
}
}
var mu sync.Mutex
inputsAsked := map[string]int{} // by candidate and node
var drops []string
respond := func(subject string, f func(m *nats.Msg) any) {
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
node := strings.Split(m.Subject, ".")[5]
var in nodeengine.ComposeCandidateInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
inputsAsked[in.Candidate+" "+node]++
mu.Unlock()
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
})
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
var in nodebuildgo.DropInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
drops = append(drops, in.ID)
mu.Unlock()
return map[string]any{}
})
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
ended := make(chan meshwants.StepEnded, 32)
sub, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data, &e)
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
subscribed(t, nc)
from := b.journal.mark()
_, stop := b.startStoppable(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
queue, _ := buildstep.AskStream(buildstep.AskGo)
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
step := func(id, module, run string, rerun bool) {
t.Helper()
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
if run != "" {
s.Pipeline = "run-" + run + "." + module
}
if rerun {
s.By = "op@n1"
}
raw, _ := json.Marshal(s)
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Data = raw
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
}
type asked struct {
ask buildstep.StepAsk
machine string
}
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
asks := func(n int) []asked {
t.Helper()
var out []asked
for range n {
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
if err != nil {
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
out = append(out, asked{a, req.Machines[0]})
}
return out
}
noMoreAsks := func(what string) {
t.Helper()
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
}
}
say := func(event string, node string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
t.Fatal(err)
}
}
answer := func(a asked, machine string) {
t.Helper()
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
}
ends := func(n int, what string) map[string]meshwants.StepEnded {
t.Helper()
out := map[string]meshwants.StepEnded{}
for len(out) < n {
select {
case e := <-ended:
out[e.ID] = e
case <-time.After(20 * time.Second):
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
}
}
return out
}
runKeys := func(candidate string) []string {
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
var out []string
for _, k := range all {
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
out = append(out, k)
}
}
return out
}
// readers waits, through a watch of the key, until the run of key has n readers.
readers := func(key string, n int) {
t.Helper()
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
select {
case e := <-w.Updates():
var run meshwants.CompositionRun
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
return
}
case <-deadline:
t.Fatalf("the run %s has not %d readers", key, n)
}
}
}
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
composes := `{"composes":true,"blamed":[]}`
// One candidate, three steps, two machines: two runs.
for _, m := range []string{"a", "b", "c"} {
step("step-"+m, m, "", false)
}
first := asks(2)
noMoreAsks("three steps of one candidate")
for _, a := range first {
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
}
got := ends(3, "the three steps")
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
t.Fatalf("the steps of one candidate: %+v", got)
}
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
}
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
step("step-r1", "a", "r1", false)
for _, a := range asks(2) {
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
}
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
t.Fatalf("a step of another candidate: %+v", e)
}
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
from = b.journal.mark()
step("step-r2", "a", "r2", false)
bound := asks(2)
var n1 asked
for _, a := range bound {
if a.machine == "n1" {
n1 = a
}
}
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
for range 3 { // past the 2 s bound, each beat within it
select {
case e := <-ended:
t.Fatalf("a run saying its progress ended: %+v", e)
case <-time.After(time.Second):
}
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
}
e := ends(1, "the bounds")["step-r2"]
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
t.Fatalf("past the bounds: %+v", e)
}
mu.Lock()
dropped := len(drops)
mu.Unlock()
if dropped != 2 {
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
}
answer(n1, composes) // heard after the bound: ignored and journaled
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
step("step-a2", "a", "", true)
again := asks(1)
noMoreAsks("a rerun of a")
if again[0].machine != "n2" {
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
}
answer(again[0], composes)
moved := ends(2, "the rerun")
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
}
// A superseded step leaves the runs; its other reader goes on.
step("step-r3a", "a", "r3", false)
step("step-r3b", "b", "r3", false)
sup := asks(2)
readers(runKeyOf("run-r3", "n1"), 2)
readers(runKeyOf("run-r3", "n2"), 2)
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}})
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
t.Fatal(err)
}
for _, a := range sup {
answer(a, composes)
}
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the other reader: %+v", got)
}
select {
case e := <-ended:
t.Fatalf("a superseded step said its end: %+v", e)
case <-time.After(200 * time.Millisecond):
}
// Restarted during a run: the new assigner takes it up by its ask.
step("step-r4", "c", "r4", false)
restart := asks(2)
stop()
for _, a := range restart {
answer(a, composes)
}
from = b.journal.mark()
b.startStoppable(t)
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
t.Fatalf("the step taken up: %+v", e)
}
// compositions-drop: the readers ended naming the drop, the key deleted.
step("step-r5", "a", "r5", false)
asks(2)
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
}
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
t.Fatalf("a reader of the dropped run: %+v", e)
}
if keys := runKeys("run-r5"); len(keys) != 1 {
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
}
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
for i := range 2 {
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
t.Fatal(err)
}
}
step("step-r6", "a", "r6", false)
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
t.Fatalf("at the cap: %+v", e)
}
}
@@ -69,11 +69,13 @@ import (
type bed struct {
// seatListRead is closed once the assigner read the seat list (serveSeatList).
seatListRead <-chan struct{}
srv *server.Server
js jetstream.JetStream
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
// settings are the files of settings a test sets, by the variable naming each (as the manifest's resources fill them).
settings map[string]string
srv *server.Server
js jetstream.JetStream
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
// placed is, by node, the modules the bed's assignments names on it (place).
placed map[string][]string
// journal is what the programs the bed starts write in their journal.
@@ -301,6 +303,13 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": settle}
for name, value := range b.settings {
file := filepath.Join(b.dir, name)
if err := os.WriteFile(file, []byte(value), 0o600); err != nil {
t.Fatal(err)
}
env[name] = file
}
ctx, stop := context.WithCancel(b.ctx)
done := make(chan error, 1)
go func() {