novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Build each build module's part of a fingerprint as one ask on its seat's stream, named in the key before it is put, so the builder chooses no node, holds no capacity and never asks twice (issue 1887 A4a, A4b, A4c, ADR 0525 §1-§6)

Browse Source
This commit is contained in:
jschoubben
2026-10-23 15:57:03 +02:00
parent 4c98841d2a
commit 9f27e2b1f8
13 changed files with 379 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+95 -119
View File
@@ -22,6 +22,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -47,13 +48,12 @@ const (
fp = "src2:3333333333333333333333333333333333333333333333333333333333333333"
)
// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on
// mesh.mod.<module>.call.<tool>.<node>.<caller> answers at once with the step and its key, and the step's end is written to
// the key <node>.<id> of the module's steps bucket, some milliseconds later.
// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls
// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that
// answer as the real tools do, ending the step on <node>.<ask id> of its steps bucket.
type fakeBuildModules struct {
js jetstream.JetStream
ctx context.Context
n atomic.Int64
failBuild atomic.Bool
// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
refusePush atomic.Bool
@@ -75,76 +75,68 @@ func digestOf(artifact string) string {
return "sha256:" + hex.EncodeToString(sum[:])
}
func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
var prefixOf = map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_", "mesh-build-image": "build_image_"}
// step is one tool's answer, as the real module's.
func (f *fakeBuildModules) step(module, name string, args map[string]any) (any, error) {
tool := prefixOf[module] + name
f.mu.Lock()
f.calls[module+"."+tool]++
if name == "fetch" {
f.fetches = append(f.fetches, args)
} else {
f.builds = append(f.builds, args)
}
f.mu.Unlock()
switch {
case name == "fetch":
merge := ""
if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult)
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
}
return json.RawMessage(`{"workspace":"w` + buildstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil
case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs
return nil, fmt.Errorf("go build of %v: boom", args["artifact"])
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
d := digestOf(fmt.Sprint(args["artifact"]))
return json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`), nil
case tool != "build_image_build" && f.refusePush.Load():
return nil, fmt.Errorf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"])
case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1)
d := digestOf(fmt.Sprint(args["artifact"]))
return json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://` + filepath.Base(fmt.Sprint(args["module"])) + `/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`), nil
case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference
d := digestOf(fmt.Sprint(args["artifact"]))
return json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/imaged/server@` + d + `"}`), nil
default:
return json.RawMessage(`{"digest":"` + digestOf(fmt.Sprint(args["artifact"])) + `"}`), nil
}
}
// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on nodes.
func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
if err != nil {
t.Fatal(err)
}
sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) {
tokens := strings.Split(m.Subject, ".") // mesh mod <module> call <tool> <node> <caller>
tool, node := tokens[4], tokens[5]
f.mu.Lock()
f.calls[module+"."+tool]++
f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
if strings.HasSuffix(tool, "_fetch") {
f.mu.Lock()
f.fetches = append(f.fetches, args)
f.mu.Unlock()
} else {
f.mu.Lock()
f.builds = append(f.builds, args)
f.mu.Unlock()
}
id := fmt.Sprintf("s%d", f.n.Add(1))
key, _ := delivery.StepKey(node, id)
put := func(s delivery.Step) {
data, err := delivery.EncodeStep(s)
if err == nil {
_, err = steps.Put(f.ctx, key, data)
}
if err != nil {
t.Errorf("writing step %s: %v", key, err)
}
}
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()}
put(step)
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node})
_ = m.Respond(answer)
go func() {
time.Sleep(30 * time.Millisecond)
step.Ended = time.Now().UTC()
switch {
case strings.HasSuffix(tool, "_fetch"):
merge := ""
if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult)
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
}
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`)
case f.failBuild.Load() && tool != "build_image_copy": // every node's build fails: one node builds a fingerprint
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"])
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
d := digestOf(fmt.Sprint(args["artifact"]))
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"artifact":"`+fmt.Sprint(args["artifact"])+`","digest":"`+d+`","reference":"registry.example/`+fmt.Sprint(args["module"])+`/`+fmt.Sprint(args["artifact"])+`@`+d+`","pushed":true}`)
case tool != "build_image_build" && f.refusePush.Load():
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"])
case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1)
d := digestOf(fmt.Sprint(args["artifact"]))
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"artifact-store://`+filepath.Base(fmt.Sprint(args["module"]))+`/`+fmt.Sprint(args["artifact"])+`/blobs/`+d+`"}`)
case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference
d := digestOf(fmt.Sprint(args["artifact"]))
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"registry.example/imaged/server@`+d+`"}`)
default:
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+digestOf(fmt.Sprint(args["artifact"]))+`"}`)
}
put(step)
}()
})
if err != nil {
language := strings.TrimPrefix(module, "mesh-build-")
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
put := func(key string, value any) (uint64, error) {
raw, _ := value.(json.RawMessage)
return kv.Put(f.ctx, key, raw)
}
call := func(name string) buildstep.StepFunc {
return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) }
}
steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
for _, node := range nodes {
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
go func() { _ = p.Pull(f.ctx) }()
}
}
type bed struct {
@@ -154,8 +146,6 @@ type bed struct {
candidates jetstream.KeyValue
builds jetstream.KeyValue
tools map[string]jetstream.KeyValue
presence jetstream.KeyValue
have jetstream.KeyValue
instances jetstream.KeyValue
stop func() // stops the builder started last
fake *fakeBuildModules
@@ -228,20 +218,11 @@ func startBed(t *testing.T) *bed {
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
presence := mk(delivery.PresenceBucket, delivery.PresenceHistory)
b.presence, b.have = presence, mk(delivery.HaveBucket, delivery.HaveHistory)
b.instances = mk(instancesBucket, 1)
for _, node := range []string{"n1", "n2"} {
b.setOnline(t, node, true)
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
b.setHealth(t, node, module, delivery.HealthOK)
}
}
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
b.fake.serve(t, ctl, "mesh-build-go")
b.fake.serve(t, ctl, "mesh-build-node")
b.fake.serve(t, ctl, "mesh-build-image")
b.setHealth(t, "n1", "mesh-build-image", delivery.HealthOK) // the image build module runs on n1 only
b.fake.serve(t, "mesh-build-go", "n1", "n2")
b.fake.serve(t, "mesh-build-node", "n1", "n2")
b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1")
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
@@ -249,28 +230,6 @@ func startBed(t *testing.T) *bed {
return b
}
// setOnline writes the watchman's presence of a node.
func (b *bed) setOnline(t *testing.T, node string, online bool) {
t.Helper()
data, _ := delivery.EncodePresence(delivery.Presence{Online: online})
if _, err := b.presence.Put(b.ctx, node, data); err != nil {
t.Fatal(err)
}
}
// setHealth writes the node-engine's have of a build module on a node, running with this health.
func (b *bed) setHealth(t *testing.T, node, module string, health delivery.Health) {
t.Helper()
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: health})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveKey(node, module)
if _, err := b.have.Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
}
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
t.Helper()
@@ -433,20 +392,8 @@ func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
}
}
// TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed: the node-engine's have of mesh-build-go on n2 says failed, so only n1 builds.
func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) {
b := startBed(t)
b.setHealth(t, "n2", "mesh-build-go", delivery.HealthFailed)
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
if !strings.Contains(v.Step, "on n1") {
t.Errorf("build = %+v, want it built on n1 alone", v)
}
}
// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails on one node, so the
// key is failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails, so the key is
// failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
// the key again as a new revision of the same key, now done.
func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
b := startBed(t)
@@ -742,3 +689,32 @@ func TestABuilderWhoseBuildModulesAnsweredReferencesSaysItAndStoppedSaysNothing(
t.Fatalf("the stopped builder still says: %v", err)
}
}
// 1887/AC1, AC2 on the real bus: the builder puts the trunk fingerprint's build as asks on the build seats' delivery
// streams and calls no node's tool; the build modules that pulled them wrote the steps, and the key is done naming the
// references.
func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) {
b := startBed(t)
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
defer sub.Close()
var calls atomic.Int64
s, err := sub.Subscribe("mesh.mod.*.call.>", func(*nats.Msg) { calls.Add(1) })
if err != nil {
t.Fatal(err)
}
defer s.Unsubscribe()
b.start(t, "n1", nil)
v := b.await(t, "netcheck", isState(delivery.BuildDone))
for _, a := range v.Artifacts {
if a.Reference == "" {
t.Errorf("artifact %s names no reference", a.Name)
}
}
if n := calls.Load(); n != 0 {
t.Errorf("%d tool calls reached a build module", n)
}
}