The assigner's whole tests no longer assume wants follow the trunk's fingerprint: a runner's pipeline-ended names the movers and versions, the wants are written tier by tier on running, and pipeline-wanted says running at the end. Test 113's cases: a provider and its consumer in tiers, the acting part's build and the old holds ignored, a failed pipeline's version and an older end heard late writing nothing, a placement's build reverting the want, tier-late after tier-bound with no later tier written, and a node not stating want:version held awaiting its hand placement. Test 117's cases run the step validate whole: a refused manifest, a pass and a cycle. The first views answer every account (no callers), and tier-bound's unit is a variable the whole test shortens. Builds: design 47 H §774, §750, §751, §772, §745, §752; acceptance tests 113, 117; lead answers H-A-10, H-A-37, H-A-40, H-A-41, H-A-56, H-A-73, H-A-75, H-A-99, H-A-108 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
375 lines
17 KiB
Go
375 lines
17 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
|
|
|
"mesh-delivery-assigner/internal/buckets"
|
|
)
|
|
|
|
// Design 47 G §354, §22, §63, §64, §389, A-240, A-252, A-254: the assigner started whole takes the runner's composition step of a
|
|
// module placed on three machines, one of them away: it asks the two others' node-engines for their inputs, puts one run ask per
|
|
// machine on the Go build seat's check queue under its name, its program the node-engine build each machine runs; the stand-in
|
|
// build seat reads each request by the seat mesh-wants' verb input in chunks and says its step-ended with the composer's verdict;
|
|
// the step's step-ended fails naming the machine that does not compose, warns of the machine away, and no input stays kept.
|
|
func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) {
|
|
b := startBed(t, "")
|
|
nc, err := nats.Connect(b.srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
const engineCommit = "1111111111111111111111111111111111111111"
|
|
// The assigner's durable on EVENTS hears, beside the bed's, the build seat's step-ended and the watchman's machine-away and
|
|
// machine-back, as the grants part derives them from its consumes; what each node runs it hears on its kept stream (the bed's).
|
|
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
|
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
|
nodebuildgo.SubjectStepEndedAll, meshwatchdogs.SubjectMachineAwayAll, meshwatchdogs.SubjectMachineBackAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, node := range []string{"n1", "n2"} {
|
|
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
|
|
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
awaitKept(t, b, nodeengine.StreamRunning, 2)
|
|
away, _ := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: "n3", At: time.Now().UTC(), LastHeard: time.Now().UTC()})
|
|
if _, err := b.js.Publish(b.ctx, meshwatchdogs.SubjectMachineAway("n3"), away); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The trunk's catalog half holds the module and the node-engine, the module claiming the seat node-engine.
|
|
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: engineCommit, Path: ".",
|
|
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
|
|
b.cat.mu.Lock()
|
|
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine,
|
|
"m": {Module: "m", Fp: "src2:m1", Repository: "novox/m", Commit: engineCommit, Path: ".", Manifest: map[string]any{"module": "m", "version": "0.1"}}}
|
|
b.cat.rev[trunkBranch] = 3
|
|
b.cat.mu.Unlock()
|
|
for _, node := range []string{"n1", "n2", "n3"} {
|
|
b.place(t, node, "m")
|
|
}
|
|
// The node-engines of n1 and n2 answer compose-candidate with their inputs; n3 is never asked.
|
|
asked := make(chan string, 8)
|
|
engines, err := nc.Subscribe("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) {
|
|
node := strings.Split(m.Subject, ".")[5]
|
|
asked <- node
|
|
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"candidate": "main.3", "node": node, "verdict": "composed", "input": map[string]any{"node": node},
|
|
"modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}, "node": node})
|
|
_ = m.Respond(raw)
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = engines.Unsubscribe() })
|
|
// mesh-catalog's entry answers the node-engine build's entry by its fingerprint.
|
|
entries, err := nc.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
|
|
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: engine}, "node": "n0"})
|
|
_ = m.Respond(raw)
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = entries.Unsubscribe() })
|
|
subscribed(t, nc)
|
|
// The runner hears the step's end.
|
|
ended := make(chan meshwants.StepEnded, 1)
|
|
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
|
|
var e meshwants.StepEnded
|
|
_ = json.Unmarshal(m.Data, &e)
|
|
ended <- e
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = ends.Unsubscribe() })
|
|
subscribed(t, nc)
|
|
|
|
from := b.journal.mark()
|
|
b.start(t)
|
|
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
|
|
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
|
|
|
|
select {
|
|
case <-b.seatListRead: // the seat server admits a caller of input once the seat list is read (A-278)
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatal("the assigner did not read the seat list within 30 s")
|
|
}
|
|
step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
|
|
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
|
|
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
|
|
msg.Header.Set("x-event-id", "step-ask-1")
|
|
msg.Data = step
|
|
if err := nc.PublishMsg(msg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
|
|
// its end with the composer's verdict: n1 composes, n2 does not.
|
|
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
|
// A-163: the assigner asks under its seat's own name.
|
|
subject := "mesh.seat.node-build-go.accept.check.mesh-wants"
|
|
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
programs := map[string]string{}
|
|
for range 2 {
|
|
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
|
|
if err != nil {
|
|
t.Fatalf("no run ask is put: %v", err)
|
|
}
|
|
_ = got.Ack()
|
|
a, err := buildstep.DecodeStepAsk(got.Data())
|
|
if err != nil || a.Run == nil {
|
|
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
|
|
}
|
|
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
|
if err != nil || len(req.Machines) != 1 {
|
|
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
|
|
}
|
|
if len(req.Seats) == 0 { // the revision names no definition: the request carries the trunk seat list's (A-467)
|
|
t.Errorf("the request of %s carries no seat definition", a.ID)
|
|
}
|
|
node := req.Machines[0]
|
|
programs[node] = a.Run.Program.Commit
|
|
m := composer.Machine{Composes: true}
|
|
if node == "n2" {
|
|
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x"}}
|
|
}
|
|
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
|
|
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(v), At: time.Now().UTC()})
|
|
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var e meshwants.StepEnded
|
|
select {
|
|
case e = <-ended:
|
|
case <-b.ctx.Done():
|
|
t.Fatal("the composition step said no end")
|
|
}
|
|
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
|
|
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
|
|
e.Warnings[1] != "n2: the composer names no culprits" {
|
|
t.Fatalf("the step ended %+v", e)
|
|
}
|
|
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
|
|
t.Errorf("the composers were built at %v, want the node-engine build each machine runs", programs)
|
|
}
|
|
close(asked)
|
|
for node := range asked {
|
|
if node == "n3" {
|
|
t.Error("the machine away was asked for its input")
|
|
}
|
|
}
|
|
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
|
|
inputs := b.buckets[buckets.Inputs()]
|
|
w, err := inputs.WatchAll(b.ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = w.Stop() }()
|
|
deadline := time.After(10 * time.Second)
|
|
for {
|
|
keys, _ := inputs.Keys(b.ctx)
|
|
if len(keys) == 0 {
|
|
break
|
|
}
|
|
select {
|
|
case <-w.Updates():
|
|
case <-deadline:
|
|
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
|
|
}
|
|
}
|
|
}
|
|
|
|
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
|
|
// chunks on the call's subjects, each acknowledged.
|
|
func readInput(t *testing.T, nc *nats.Conn, in buildstep.RunInput) []byte {
|
|
t.Helper()
|
|
call := "standin" + strings.Repeat("0", 16)
|
|
chunks := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: call}
|
|
sub, err := nc.SubscribeSync(chunks.Chunks())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sub.Unsubscribe()
|
|
body, _ := json.Marshal(meshwants.InputInput{Digest: in.Digest, Call: call})
|
|
// On the call's subject naming its caller, the stand-in build seat's account: the seat server admits no unproven caller (A-179).
|
|
reply, err := nc.Request("mesh.seat.mesh-wants.call.input.n1~mesh-build-go", body, 10*time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, said, ok := buscalls.Unwrap(reply.Data)
|
|
var a meshwants.InputAnswer
|
|
if !ok || json.Unmarshal(raw, &a) != nil || a.Chunks < 1 {
|
|
t.Fatalf("the verb input answered %s: %s", reply.Data, said)
|
|
}
|
|
var out bytes.Buffer
|
|
next := func() (string, []byte, error) {
|
|
m, err := sub.NextMsg(buscalls.SeatAckWithin)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
return m.Subject, m.Data, nil
|
|
}
|
|
ack := func(subject string) error { _, err := nc.Request(subject, nil, 10*time.Second); return err }
|
|
if _, err := chunks.Receive(a.Chunks, next, ack, &out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := in.Verify(out.Bytes()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Bytes()
|
|
}
|
|
|
|
// Design 47 H §745 (acceptance test 117, the assigner's half; H-A-108), started whole on the bus: the assigner serves the step
|
|
// validate on the seat mesh-wants. A take-in whose manifest names an unknown field fails it `manifest-refused` naming it, the
|
|
// validator being the revision's own entry of the module holding the seat mesh-catalog, run on the Go build seat over the
|
|
// revision's manifests and the previous revision's; a take-in that changes a manifest with no problem passes it; a take-in whose
|
|
// entries make a cycle fails it `cycle` naming the cycle, with no validator asked.
|
|
func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T) {
|
|
b := startBed(t, "")
|
|
nc, err := nats.Connect(b.srv.ClientURL())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
const commit = "2222222222222222222222222222222222222222"
|
|
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
|
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
|
nodebuildgo.SubjectStepEndedAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commit, Path: "modules/mesh-delivery-indexer",
|
|
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"}
|
|
entry := func(module, digest string, manifest map[string]any, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
|
|
manifest["module"] = module
|
|
return meshcatalog.CatalogEntry{Module: module, Manifest: manifest, ManifestDigest: digest, Edges: edges}
|
|
}
|
|
b.cat.mu.Lock()
|
|
b.cat.rev[trunkBranch] = 9
|
|
b.cat.at = map[string]map[int64][]meshcatalog.CatalogEntry{trunkBranch: {
|
|
6: {indexer, entry("web", "d-web-1", map[string]any{"version": "1"})},
|
|
7: {indexer, entry("web", "d-web-2", map[string]any{"version": "2", "colour": "red"})},
|
|
8: {indexer, entry("web", "d-web-3", map[string]any{"version": "3"})},
|
|
9: {indexer, entry("a", "d-a", map[string]any{}, meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}),
|
|
entry("b", "d-b", map[string]any{})},
|
|
}}
|
|
b.cat.mu.Unlock()
|
|
ended := make(chan meshwants.StepEnded, 4)
|
|
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
|
|
var e meshwants.StepEnded
|
|
_ = json.Unmarshal(m.Data, &e)
|
|
ended <- e
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = ends.Unsubscribe() })
|
|
subscribed(t, nc)
|
|
from := b.journal.mark()
|
|
b.start(t)
|
|
b.journal.await(t, from, "the composition step is heard on", "the steps served", 30*time.Second)
|
|
select {
|
|
case <-b.seatListRead:
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatal("the assigner did not read the seat list within 30 s")
|
|
}
|
|
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
|
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants", AckPolicy: jetstream.AckExplicitPolicy})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ask := func(revision int64, module string, run func()) meshwants.StepEnded {
|
|
t.Helper()
|
|
id := fmt.Sprintf("validate-%d", revision)
|
|
step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module,
|
|
Branch: trunkBranch, Revision: revision, Commit: commit})
|
|
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
|
|
msg.Header.Set("x-event-id", id)
|
|
msg.Data = step
|
|
if err := nc.PublishMsg(msg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if run != nil {
|
|
run()
|
|
}
|
|
for {
|
|
select {
|
|
case e := <-ended:
|
|
if e.ID == id {
|
|
return e
|
|
}
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatalf("the validate step of revision %d said no end", revision)
|
|
}
|
|
}
|
|
}
|
|
// The stand-in Go build seat runs the validator: it refuses a manifest naming the field colour, and accepts every other.
|
|
validator := func() {
|
|
t.Helper()
|
|
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
|
|
if err != nil {
|
|
t.Fatalf("no validator run is asked: %v", err)
|
|
}
|
|
_ = got.Ack()
|
|
a, err := buildstep.DecodeStepAsk(got.Data())
|
|
if err != nil || a.Run == nil || a.Run.Program.Repository != "novox/mesh-delivery" || a.Run.Program.Commit != commit {
|
|
t.Fatalf("the validator's run ask %s: %v", got.Data(), err)
|
|
}
|
|
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
|
if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase {
|
|
t.Fatalf("the validator's request: %+v %v", req, err)
|
|
}
|
|
v := composer.Verdict{V: composer.Version}
|
|
for _, c := range req.Cases {
|
|
reads := map[string]composer.ManifestRead{}
|
|
for module, m := range c.Manifests {
|
|
reads[module] = composer.ManifestRead{Accepted: true}
|
|
if strings.Contains(string(m.Raw), "colour") {
|
|
reads[module] = composer.ManifestRead{Refused: `unknown field "colour"`}
|
|
}
|
|
}
|
|
v.Cases = append(v.Cases, composer.CaseVerdict{Name: c.Name, Manifests: reads})
|
|
}
|
|
out, _ := json.Marshal(v)
|
|
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(out), At: time.Now().UTC()})
|
|
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if e := ask(7, "web", validator); e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(e.Said, reasonManifestRefused+": ") || !strings.Contains(e.Said, `web: unknown field "colour"`) {
|
|
t.Errorf("the validate step of a manifest naming an unknown field ended %+v", e)
|
|
}
|
|
if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed {
|
|
t.Errorf("the validate step of a manifest with no problem ended %+v", e)
|
|
}
|
|
if e := ask(9, "a", nil); e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
|
|
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
|
|
}
|
|
if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 {
|
|
t.Errorf("a validator is asked for the cycle: %+v %v", info, err)
|
|
}
|
|
}
|