novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge feat/r4e-builder into feat/r4e (R4e integration)

Browse Source 
The checker's go.mod and go.sum conflicted on the mesh-sdk pin (checker lane 9cb4d59, builder lane e76a5a3): every
module touched is pinned to the sdk lane's newest head d1af87a until the final SDK head.
This commit is contained in:
jschoubben
2026-10-22 20:29:53 +02:00
parent 173becf35e 612658e788
commit 91e68ac13b
66 changed files with 2950 additions and 933 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go
+1 -1
View File
@@ -155,5 +155,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.ParseManifest([]byte(kept.JSON))
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+81 -42
View File
@@ -54,10 +54,12 @@ type fakeBuildModules struct {
ctx context.Context
n atomic.Int64
failBuild atomic.Bool
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
fetches []map[string]any
builds []map[string]any // the arguments of every build call
// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
refusePush atomic.Bool
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
fetches []map[string]any
builds []map[string]any // the arguments of every build call
}
func (f *fakeBuildModules) count(tool string) int {
@@ -119,8 +121,16 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
}
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`)
case f.failBuild.Load() && node == "n1":
case f.failBuild.Load() && tool != "build_image_copy": // every node's build fails: one node builds a fingerprint
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"])
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
d := digestOf(fmt.Sprint(args["artifact"]))
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"artifact":"`+fmt.Sprint(args["artifact"])+`","digest":"`+d+`","reference":"registry.example/`+fmt.Sprint(args["module"])+`/`+fmt.Sprint(args["artifact"])+`@`+d+`","pushed":true}`)
case tool != "build_image_build" && f.refusePush.Load():
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"])
case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1)
d := digestOf(fmt.Sprint(args["artifact"]))
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"artifact-store://`+filepath.Base(fmt.Sprint(args["module"]))+`/`+fmt.Sprint(args["artifact"])+`/blobs/`+d+`"}`)
case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference
d := digestOf(fmt.Sprint(args["artifact"]))
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"registry.example/imaged/server@`+d+`"}`)
@@ -364,7 +374,7 @@ func isState(state string) func(delivery.BuilderBuild) bool {
// TestBuilderMakesADoneKeyWithTheDigests: modules.netcheck (a Go and a TypeScript bundle) and its manifest are in the
// indexer's buckets, and two nodes report each build module's toolchain; the program writes builds.netcheck.<fp> done,
// the digests of both bundles, both build modules' toolchains, after a fetch and a build on each of the two nodes.
// the digests of both bundles, both build modules' toolchains, after a fetch and a build on one node (ADR 0478 clause 4).
func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
b := startBed(t)
digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
@@ -374,16 +384,16 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
t.Fatalf("build = %+v", v)
}
for _, a := range v.Artifacts {
if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "" {
t.Errorf("artifact = %+v, want the digest every node built and no reference", a)
if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "artifact-store://netcheck/"+a.Name+"/blobs/"+a.Digest {
t.Errorf("artifact = %+v, want the digest every node built and the reference its push answered (ADR 0498 clause 1)", a)
}
}
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
t.Errorf("toolchains = %+v", v.Toolchains)
}
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 2, "mesh-build-go.build_go_build": 2, "mesh-build-node.build_node_fetch": 2, "mesh-build-node.build_node_build": 2} {
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} {
if got := b.fake.count(tool); got != want {
t.Errorf("%s called %d times, want %d (once on each of two nodes)", tool, got, want)
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
}
}
}
@@ -400,8 +410,8 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { //
if len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files") {
t.Fatalf("build = %+v, want the archive's digest", v)
}
if n := b.fake.count("mesh-build-go.build_go_build"); n < 2 || n%2 != 0 { // each instance that built, on two nodes
t.Errorf("build_go_build was called %d times, want two for each instance that built", n)
if n := b.fake.count("mesh-build-go.build_go_build"); n < 1 { // each instance that built, on one node
t.Errorf("build_go_build was called %d times, want one for each instance that built", n)
}
}
@@ -413,32 +423,7 @@ func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") {
t.Errorf("build = %+v, want done on two nodes naming no toolchain", v)
}
}
// TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline: n2 is offline, so the key is done on n1 alone and marked;
// n2 coming online makes the builder build it again on both nodes, as a new revision (ADR 0478 clause 4).
func TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline(t *testing.T) {
b := startBed(t)
b.setOnline(t, "n2", false)
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
one := b.await(t, "ca-trust", isState(delivery.BuildDone))
if !strings.HasPrefix(one.Step, "one node (") {
t.Fatalf("build = %+v, want it marked as built on one node", one)
}
key := mustKey(t, "ca-trust")
e, err := b.builds.Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
b.setOnline(t, "n2", true)
two := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool {
return v.State == delivery.BuildDone && !strings.HasPrefix(v.Step, "one node (")
})
if e2, _ := b.builds.Get(b.ctx, key); e2.Revision() <= e.Revision() || !strings.Contains(two.Step, "n1 and n2") {
t.Errorf("build = %+v at revision %d (was %d), want a new revision built on n1 and n2", two, e2.Revision(), e.Revision())
t.Errorf("build = %+v, want done naming no toolchain", v)
}
}
@@ -449,7 +434,7 @@ func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) {
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
if !strings.Contains(v.Step, "on one node, n1") {
if !strings.Contains(v.Step, "on n1") {
t.Errorf("build = %+v, want it built on n1 alone", v)
}
}
@@ -463,7 +448,7 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFing
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
if !strings.Contains(v.Error, "n1") || !strings.Contains(v.Error, "boom") {
if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
}
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
@@ -500,8 +485,8 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil {
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2")
}
if n := b.fake.count("mesh-build-go.build_go_build"); n != 2 {
t.Errorf("build_go_build was called %d times, want 2 (the new fingerprint on two nodes only)", n)
if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 {
t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n)
}
}
@@ -674,3 +659,57 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
t.Errorf("builds = %v, want three, two of them with publish (the trunk's two fingerprints)", b.fake.builds)
}
}
// TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain: with the artifact store refusing pushes, the key of an
// archive module ends failed opening "not stored:"; once it takes them, a new modules value builds the key again, done, its
// every artifact naming a reference (issue 1587, B8, ADR 0498 clauses 1, 2 and 8).
func TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain(t *testing.T) {
b := startBed(t)
b.fake.refusePush.Store(true)
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
if !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") {
t.Fatalf("build = %+v, want failed opening %q with the store's words", v, delivery.NotStored)
}
b.fake.refusePush.Store(false)
b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
v = b.await(t, "ca-trust", isState(delivery.BuildDone))
if ok, missing := v.Referenced(); !ok {
t.Errorf("build = %+v, artifacts naming no reference: %v", v, missing)
}
}
// TestAPinnedUpstreamIsCopiedByTheImageBuildModule: hello-web's manifest (captured from mesh-catalog 49caf17f) names an
// upstream image pinned by its digest and an archive; the upstream is copied on n1, where mesh-build-image runs, with no
// fetch of the image module, and the key ends done with both references (issue 1587, B8, ADR 0498 clause 4).
func TestAPinnedUpstreamIsCopiedByTheImageBuildModule(t *testing.T) {
b := startBed(t)
b.moduleCommit(t, "hello-web", "catalogue-49caf17f-hello-web.module.json", "modules/hello-web", nil, commit)
b.start(t, "n1", nil)
v := b.await(t, "hello-web", isState(delivery.BuildDone))
if ok, missing := v.Referenced(); !ok {
t.Fatalf("build = %+v, artifacts naming no reference: %v", v, missing)
}
if n := b.fake.count("mesh-build-image.build_image_copy"); n != 1 {
t.Errorf("build_image_copy called %d times, want once", n)
}
if n := b.fake.count("mesh-build-image.build_image_fetch"); n != 0 {
t.Errorf("build_image_fetch called %d times, want none for a copy", n)
}
}
// TestAManifestFallingBackToAPredefinedKeyIsBuilt: claude-code's manifest (mesh-catalog 835bf22b) has a config value falling
// back to a predefined key; the builder, which holds no published list of them, reads it and builds the key done (issue
// 1751, mesh-sdk catalogue.ParseManifestForInputs).
func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) {
b := startBed(t)
b.module(t, "claude-code", "catalogue-835bf22b-claude-code.module.json", "modules/claude-code")
b.start(t, "n1", nil)
v := b.await(t, "claude-code", func(v delivery.BuilderBuild) bool {
return v.State == delivery.BuildDone || v.State == delivery.BuildFailed
})
if v.State != delivery.BuildDone {
t.Errorf("build = %+v, want done: the manifest is read", v)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
-3
View File
@@ -35,8 +35,6 @@ const (
// defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a
// build compiles).
defaultStepBound = 30 * time.Minute
// nodesPerFingerprint is how many build nodes build each fingerprint until release R4e (ADR 0478 clause 4).
nodesPerFingerprint = 2
)
func main() {
@@ -178,7 +176,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Health: have,
Places: &buildrun.Places{Capacity: capacity},
Manifests: manifestReader{manifests},
Nodes: nodesPerFingerprint,
Logf: logger.Printf,
}
// The toolchains first, so a build that starts from the first modules value finds the nodes.
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json
Vendored
+82
View File
@@ -0,0 +1,82 @@
{
"module": "hello-web",
"slug": "hello",
"version": "1",
"capabilities": [
"container-runtime"
],
"requires": [
"route"
],
"contributes": {
"route": {
"label": "hello",
"endpoint": "web"
}
},
"binds": {
"route": "${dir:state}/route.json"
},
"listens": [
{
"name": "web",
"port": 8080,
"protocol": "tcp",
"from": "mesh",
"why": "the demo web page; only the route-proxy reaches it, and the public name is a route grant"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": ".",
"unknown": "program"
},
{
"id": "page",
"type": "file",
"path": "${dir:state}/index.html",
"mode": "0644",
"template": "files/index.html"
},
{
"id": "net",
"type": "network",
"name": "hello-web"
},
{
"id": "server",
"type": "container",
"name": "hello-web",
"network": "hello-web",
"ports": [
"8080"
],
"volumes": [
"${dir:state}/index.html:/www/index.html:ro"
],
"args": [
"sh",
"-c",
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done"
],
"artifact": "server"
}
],
"build": {
"artifacts": [
{
"name": "server",
"kind": "upstream",
"from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json
Vendored
+249
View File
@@ -0,0 +1,249 @@
{
"module": "claude-code",
"version": "1",
"moved": true,
"slug": "agent",
"capabilities": [
"package-manager"
],
"requires": [
"mcp-endpoint",
{
"name": "gitea-read",
"optional": true
}
],
"binds": {
"mcp-endpoint": "${dir:state}/mcp-endpoint.json"
},
"secrets": {
"gitea-read": "${dir:state}/gitea-read.secret"
},
"uses": [
"operator-channel"
],
"state": [
"servers",
"holdings",
"config",
"proposals",
"questions",
"memory",
"waits"
],
"reads": [
"claude-licence-manager.bindings"
],
"tools": [
"claude_code_status",
"claude_code_render",
"claude_code_guard",
"claude_code_pull",
"claude_code_grant",
"claude_code_add_api_key",
"claude_code_registrations",
"claude_code_memory",
"claude_code_mcp_list",
"claude_code_mcp_register",
"claude_code_mcp_unregister",
"claude_code_skill_list",
"claude_code_skill_register",
"claude_code_skill_unregister",
"claude_code_agent_list",
"claude_code_agent_register",
"claude_code_agent_unregister",
"claude_code_command_list",
"claude_code_command_register",
"claude_code_command_unregister",
"claude_code_hook_list",
"claude_code_hook_register",
"claude_code_hook_unregister",
"claude_code_output_style_list",
"claude_code_output_style_register",
"claude_code_output_style_unregister",
"claude_code_instructions_list",
"claude_code_instructions_register",
"claude_code_instructions_unregister",
"claude_code_instructions_propose",
"claude_code_proposals",
"claude_code_ask",
"claude_code_answer",
"claude_code_wait",
"claude_code_settings_get",
"claude_code_settings_set",
"claude_code_settings_clear",
"claude_code_permission_add",
"claude_code_permission_remove",
"claude_code_config_list",
"claude_code_config_show",
"claude_code_config_status",
"claude_code_config_import",
"claude_code_home_show",
"claude_code_home_remove",
"claude_code_home_removed",
"claude_code_home_restore",
"claude_code_judge",
"claude_code_judged"
],
"data": {
"own": [
{
"id": "agent-home",
"path": "${dir:agent-home}",
"class": "valuable",
"why": "the operator's agent's own files: its memory, history and projects"
}
]
},
"config": {
"role": {
"type": "string",
"description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.",
"trusted": true,
"default": ""
},
"mcp_servers": {
"type": "object",
"description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.",
"trusted": true,
"default": {}
},
"managed_settings": {
"type": "object",
"description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.",
"trusted": true,
"predefined": "agent-managed-settings",
"default": {}
},
"instructions": {
"type": "object",
"description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
},
"output_styles": {
"type": "object",
"description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
},
"skills": {
"type": "object",
"description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.",
"trusted": true,
"default": {}
},
"commands": {
"type": "object",
"description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
},
"agents": {
"type": "object",
"description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
}
},
"resources": [
{
"id": "package",
"type": "package",
"package": "claude-code"
},
{
"id": "managed",
"type": "directory",
"path": "/etc/claude-code",
"mode": "0755",
"unknown": "program"
},
{
"id": "start",
"type": "file",
"path": "/usr/local/bin/claude-agent",
"mode": "0755",
"template": "files/claude-agent"
},
{
"id": "agent-account-name",
"type": "file",
"path": "/etc/claude-code/agent-account",
"mode": "0644",
"template": "files/agent-account"
},
{
"id": "agent-account",
"type": "user",
"name": "${machine:agent-account}",
"home": "${machine:agent-home}",
"root": "${machine:agent-root}"
},
{
"id": "agent-home",
"type": "directory",
"path": "${machine:agent-home}/.claude",
"mode": "0700",
"owner": "${machine:agent-account}",
"unknown": "module"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "${machine:account}",
"place": ".",
"unknown": "program"
},
{
"id": "facts",
"type": "file",
"path": "${dir:state}/facts.json",
"mode": "0600",
"owner": "${machine:account}",
"template": "files/facts.json"
},
{
"id": "settings",
"type": "file",
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "${machine:account}",
"takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"],
"merge": "json",
"template": "files/settings.json"
}
],
"items": [
{
"collection": "shell:zsh:normal",
"template": "files/zsh-normal"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/claude-code",
"binary": "claude-code",
"loads": [
"claude-code"
],
"env": {
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
}
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-builder/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-builder/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-builder/internal/build/candidates.go
+4
View File
@@ -69,6 +69,10 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
}
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
Toolchains: b.agreedFor(ctx, cm.Manifest)}
if len(cm.Paths) > 0 {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
modules/mesh-delivery-builder/internal/build/candidates_test.go
+3 -3
View File
@@ -54,8 +54,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
t.Fatalf("build = %+v, want the candidate's fingerprint built", v)
}
fetches := r.bm.callsOf("build_go_fetch")
if len(fetches) != 2 { // one on each of two nodes
t.Fatalf("%d fetches, want one per node", len(fetches))
if len(fetches) != 1 { // one node per fingerprint (ADR 0478 clause 4)
t.Fatalf("%d fetches, want one", len(fetches))
}
f := fetches[0].body.(map[string]any)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
@@ -67,7 +67,7 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
// Two heads carrying one fingerprint are one build.
r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY))
r.b.Wait()
if n := len(r.bm.callsOf("build_go_fetch")); n != 2 {
if n := len(r.bm.callsOf("build_go_fetch")); n != 1 {
t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n)
}
}
modules/mesh-delivery-builder/internal/build/claim.go
+72 -36
View File
@@ -4,10 +4,12 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given.
@@ -37,48 +39,71 @@ type Claim struct {
Build delivery.BuilderBuild
}
// OneNodePrefix opens the Step of a done build that a build module built on one node only, because no second node
// qualified (ADR 0478 clause 4); the build modules that did so follow in parentheses, then ": ". Claim reads it to build
// the key again, as a new revision, once a second node qualifies.
const OneNodePrefix = "one node ("
// OneNodeModules are the build modules a done build built on one node only, or none when it is not such a build.
func OneNodeModules(b delivery.BuilderBuild) []string {
if b.State != delivery.BuildDone || !strings.HasPrefix(b.Step, OneNodePrefix) {
return nil
}
list, _, ok := strings.Cut(strings.TrimPrefix(b.Step, OneNodePrefix), "): ")
if !ok || list == "" {
return nil
}
return strings.Split(list, ", ")
}
// ClaimOptions say what the builder knows about the moment of a claim.
type ClaimOptions struct {
// TakeOver takes a key found building whoever holds it: the builder has just started and looks at the key for the first
// time, and a build left building by a builder that is gone, on this node or another, is nobody's (ADR 0478 clause 6).
TakeOver bool
// MoreNodes says whether one of these build modules, which built a done key on one node only, has a second node that
// qualifies now and the key may be built again (it is not held back after a failed second build, issue 1664). Nil means
// none does.
MoreNodes func(modules []string) bool
// ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken
// again only then. Nil means none does.
ImageBuilder func() bool
// Trunk is whether the trunk's record names this fingerprint. A done key whose image has no reference was built as a
// candidate's, which never pushes (ADR 0487 clause 12); on the trunk it is taken again to publish the image, when an image
// builder runs (ImageBuilder).
// Trunk is whether the trunk's record names this fingerprint. A done key whose image or package has no reference was
// built as a candidate's, which never pushes or publishes (ADR 0487 clause 12, ADR 0498 clauses 5 and 6); on the trunk it
// is taken again to publish them, an image only when an image builder runs (ImageBuilder).
Trunk bool
// Published says whether a package version `<name>@<version>` is published: one of the builder's done keys names it as
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
// clause 6). Nil means none is.
Published func(ref string) bool
}
// Unpublished is whether a done build holds an image built without a push: it has no reference.
func Unpublished(b delivery.BuilderBuild) bool {
// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has
// no reference (a candidate's, ADR 0487 clause 12, ADR 0498 clauses 5 and 6).
func Unpublished(b delivery.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 }
// UnpublishedModules are the build modules that publish what a done build holds unpublished: the image build module for
// an image, the Node build module for a package; sorted, none when it holds nothing unpublished.
func UnpublishedModules(b delivery.BuilderBuild) []string {
if b.State != delivery.BuildDone {
return nil
}
var image, pkg bool
for _, a := range b.Artifacts {
image = image || (a.Kind == catalogue.ArtifactImage && a.Reference == "")
pkg = pkg || (a.Kind == catalogue.ArtifactPackage && a.Reference == "")
}
var out []string
if image {
out = append(out, ImageBuildModule)
}
if pkg {
out = append(out, buildrun.NodeBuildModule)
}
sort.Strings(out)
return out
}
// publishable are the build modules of UnpublishedModules that can publish now: the image build module only while a node
// runs it (imageBuilder).
func publishable(b delivery.BuilderBuild, imageBuilder func() bool) []string {
var out []string
for _, bm := range UnpublishedModules(b) {
if bm != ImageBuildModule || (imageBuilder != nil && imageBuilder()) {
out = append(out, bm)
}
}
return out
}
// Unreferenced is whether a done build holds a bundle, an archive or an upstream copy that names no reference: a key done
// before R4e, whose artifacts stayed on their build nodes (ADR 0498 clause 8). An artifact of no kind (a key written before
// kinds) is one too. Claim takes such a key again, once: its new revision names them.
func Unreferenced(b delivery.BuilderBuild) bool {
if b.State != delivery.BuildDone {
return false
}
for _, a := range b.Artifacts {
if a.Kind == "image" && a.Reference == "" {
if a.Reference == "" && (a.Kind == "" || stored(a.Kind)) {
return true
}
}
@@ -87,13 +112,19 @@ func Unpublished(b delivery.BuilderBuild) bool {
// Claim takes the build of module at fingerprint, or answers nil when it is not this builder's to build now:
// - no key: it is created building, by this builder; a create that finds the key made meanwhile gets nothing;
// - a done key is not taken again (the same source is never built twice), except one built on one node only (OneNodePrefix)
// when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its
// builder writes the new revision when the second build ends (ADR 0478 clause 4);
// - a done key is not taken again (the same source is never built twice), except one whose bundle, archive or upstream
// copy names no reference (Unreferenced, a key done before R4e: ADR 0498 clause 8), which is claimed building as a new
// revision and built whole, and one whose image or package was built unpublished as a candidate's once the trunk
// names it (ClaimOptions.Trunk): that is taken as it stands, with the done value, and its builder writes the new
// revision when the publishing build ends (ADR 0498 clause 6). From R4e one node builds each fingerprint (ADR 0478
// clause 4), so a key built on one node is never built again for a second;
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
// own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6).
func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string, opt ClaimOptions) (*Claim, error) {
@@ -115,19 +146,24 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
}
switch {
case cur.State == delivery.BuildDone:
if modules := OneNodeModules(cur); len(modules) > 0 && opt.MoreNodes != nil && opt.MoreNodes(modules) {
return &Claim{Key: key, Rev: rev, Build: cur}, nil
if Unreferenced(cur) {
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
}
if opt.Trunk && Unpublished(cur) && opt.ImageBuilder != nil && opt.ImageBuilder() {
if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 {
return &Claim{Key: key, Rev: rev, Build: cur}, nil
}
return nil, nil
case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
case cur.State == delivery.BuildFailed && (delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
return nil, nil
case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
return nil, nil
case cur.State == delivery.BuildFailed:
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
case cur.State == delivery.BuildQueued && cur.Waits != "":
if opt.Published == nil || !opt.Published(cur.Waits) {
return nil, nil // the package it waits for is not published yet
}
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
case cur.By == c.By || opt.TakeOver:
return &Claim{Key: key, Rev: rev, Build: cur}, nil
}
modules/mesh-delivery-builder/internal/build/claim_test.go
+11 -3
View File
@@ -125,9 +125,9 @@ func TestClaimLosingTheCreateRaceGetsNothing(t *testing.T) { // 1607/AC3
}
}
func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3
func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3; a key whose artifacts name references (ADR 0498 clause 8)
st := newMem()
seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Digest: digest}}})
seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/x/blobs/" + digest}}})
writes := st.writes
got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{})
if err != nil || got != nil || st.writes != writes {
@@ -148,11 +148,19 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) {
}
st = newMem()
key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: RefusedPrefix + " image (artifact x): build-agent builds it"})
key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: delivery.RefusedKind + "package of language \"python\" (artifact x)"})
writes := st.writes
if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes {
t.Errorf("a refused kind: claim = %v, %v, writes %d (was %d), want none and no write", got, err, st.writes, writes)
}
for _, e := range []string{delivery.NotPinned + "up: nats:2 names no digest", delivery.VersionTaken + "@novox/mesh-sdk@0.1.15"} { // B3, B5: ADR 0498 clauses 4, 5
st = newMem()
seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: e})
writes = st.writes
if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes {
t.Errorf("%s: claim = %v, %v, writes %d (was %d), want none until the fingerprint changes", e, got, err, st.writes, writes)
}
}
}
func TestAKeyFailedForNoImageBuilderIsTakenAgainOnlyWhenANodeHasTheModule(t *testing.T) { // 1705b
modules/mesh-delivery-builder/internal/build/followups_test.go
-37
View File
@@ -8,43 +8,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1664: a done key built on one node is not built again, over and over, while the second node's build keeps failing.
func TestASecondNodeBuildThatFailsIsNotRetriedUntilTheNodesChange(t *testing.T) { // issue 1664
r := newRig(t, goBundle)
r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies
first := r.consider(t, "modules/m")
r.bm.fails["ace/build_go_fetch"] = "ace is out of disk"
r.pres.Put("ace", presenceValue(t, true), false)
r.b.NodesChanged(context.Background()) // ace came online: one more try
r.b.Wait()
tries := len(r.bm.callsOf("build_go_fetch"))
if tries < 2 {
t.Fatalf("%d fetches: the second node was never tried", tries)
}
if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) {
t.Fatalf("the done key changed: %+v", v)
}
// Nothing about the nodes changed: a modules value, a Reconsider and a start of looking again build nothing.
for i := 0; i < 3; i++ {
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Reconsider(context.Background())
r.b.Wait()
}
if n := len(r.bm.callsOf("build_go_fetch")); n != tries {
t.Errorf("%d fetches after looking again, was %d: the failed second build was retried in a loop", n, tries)
}
// A node changing (ace goes and comes back) is a reason to try again, and ace now builds.
delete(r.bm.fails, "ace/build_go_fetch")
r.pres.Put("ace", presenceValue(t, false), false)
r.b.NodesChanged(context.Background())
r.pres.Put("ace", presenceValue(t, true), false)
r.b.NodesChanged(context.Background())
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") {
t.Errorf("build = %+v, want it built on both nodes once ace could", v)
}
}
// 1660: at start the builder closes a key left building for a fingerprint the module no longer has.
func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *testing.T) { // issue 1660
r := newRig(t, goBundle)
modules/mesh-delivery-builder/internal/build/plan.go
+31 -13
View File
@@ -5,16 +5,14 @@ package build
import (
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// RefusedPrefix opens the error of a build key that is failed for a kind no builder builds yet (ADR 0478 clause 2): such
// a key is not built again as a new revision.
const RefusedPrefix = "refused kind:"
// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that
// run it. It has no toolchain language; the builder finds its nodes from the node-engine's have.
const ImageBuildModule = "mesh-build-image"
@@ -31,6 +29,9 @@ type Task struct {
Kind string
Module string
Language string
// From is, for an upstream artifact, the reference its manifest pins, which the image build module copies (ADR 0498
// clause 4).
From string
}
// Plan is what a manifest asks the builder to build, by artifact name.
@@ -54,25 +55,34 @@ func (p Plan) Modules() []string {
// (the indexer gives an archive no language, so a fingerprint names no toolchain for it; ADR 0478 clause 1).
func (p Plan) NeedsToolchain(module string) bool {
for _, t := range p.Tasks {
if t.Module == module && t.Kind != catalogue.ArtifactArchive && t.Kind != catalogue.ArtifactImage {
if t.Module == module && t.Kind != catalogue.ArtifactArchive && t.Kind != catalogue.ArtifactImage && t.Kind != catalogue.ArtifactUpstream {
return true
}
}
return false
}
// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) it has no build
// module for.
// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) no build module
// builds. It opens delivery.RefusedKind, so its key is not built again until the fingerprint changes.
type Refused struct{ Artifact, What string }
func (r *Refused) Error() string {
return fmt.Sprintf("%s %s (artifact %s): the builder builds Go and TypeScript bundles and archives; build-agent builds it", RefusedPrefix, r.What, r.Artifact)
return fmt.Sprintf("%s%s (artifact %s): no build module builds it; the builder builds Go and TypeScript bundles, archives, images, upstream copies and TypeScript packages", delivery.RefusedKind, r.What, r.Artifact)
}
// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2): a Go bundle and an archive are built by the Go build
// module, an image by the image build module (ADR 0487), a TypeScript bundle by the Node one, each in the order of the artifacts' names. A manifest with an
// upstream or package artifact, or a bundle of another language, is refused as a whole, naming the first such artifact
// by name. A manifest with no artifact has an empty plan.
// NotPinned is the error of a plan with an upstream artifact whose reference names no digest (ADR 0498 clause 4): it opens
// delivery.NotPinned, so its key is not built again until the fingerprint changes.
type NotPinned struct{ Artifact, From string }
func (n *NotPinned) Error() string {
return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", delivery.NotPinned, n.Artifact, n.From)
}
// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2, ADR 0498 clauses 4 and 5): a Go bundle and an archive are
// built by the Go build module, an image by the image build module (ADR 0487), which also copies an upstream image pinned
// by its digest, and a TypeScript bundle or package by the Node one, each in the order of the artifacts' names. A manifest
// with an upstream artifact pinned by no digest, or a bundle or a package of another language, or a kind no build module
// builds, is refused as a whole, naming the first such artifact by name. A manifest with no artifact has an empty plan.
func PlanOf(m catalogue.Manifest) (Plan, error) {
if m.Build == nil {
return Plan{}, nil
@@ -90,8 +100,16 @@ func PlanOf(m catalogue.Manifest) (Plan, error) {
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"})
case a.Kind == catalogue.ArtifactImage:
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule})
case a.Kind == catalogue.ArtifactUpstream && !strings.Contains(a.From, "@sha256:"):
return Plan{}, &NotPinned{Artifact: a.Name, From: a.From}
case a.Kind == catalogue.ArtifactUpstream:
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule, From: a.From})
case a.Kind == catalogue.ArtifactPackage && a.Language == "typescript":
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"})
case a.Kind == catalogue.ArtifactBundle:
return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)}
case a.Kind == catalogue.ArtifactPackage:
return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("package of language %q", a.Language)}
default:
return Plan{}, &Refused{Artifact: a.Name, What: a.Kind}
}
modules/mesh-delivery-builder/internal/build/plan_test.go
+27 -19
View File
@@ -2,6 +2,8 @@ package build
import (
"errors"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strings"
"testing"
@@ -41,29 +43,35 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
}
}
func TestPlanRefusesUpstreamAndPackageByName(t *testing.T) { // 1607/AC2, ADR 0478 clause 2
for _, c := range []struct {
artifact catalogue.Artifact
kind string
}{
{catalogue.Artifact{Name: "up", Kind: "upstream"}, "upstream"},
{catalogue.Artifact{Name: "pkg", Kind: "package", Language: "typescript"}, "package"},
} {
m := manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, c.artifact)
p, err := PlanOf(m)
if err == nil {
t.Errorf("%s: plan %+v, want a refusal", c.kind, p)
continue
}
var r *Refused
if !asRefused(err, &r) || !strings.HasPrefix(err.Error(), RefusedPrefix) || !strings.Contains(err.Error(), c.kind) || !strings.Contains(err.Error(), "build-agent") {
t.Errorf("%s: error = %v, want it to open %q and name the kind and build-agent", c.kind, err, RefusedPrefix)
}
const pinned = "nats:2.11.17-alpine@sha256:3333333333333333333333333333333333333333333333333333333333333333"
func TestPlanSendsAPinnedUpstreamToTheImageBuildModulesCopyAndRefusesAnUnpinnedOne(t *testing.T) { // B3, ADR 0498 clause 4
p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned}))
if err != nil || len(p.Tasks) != 2 {
t.Fatalf("plan %+v, error %v", p, err)
}
if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != ImageBuildModule || up.From != pinned || up.Language != "" {
t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, ImageBuildModule)
}
if p.NeedsToolchain(ImageBuildModule) {
t.Error("a copy needs no toolchain")
}
_, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}))
if err == nil || !strings.HasPrefix(err.Error(), delivery.NotPinned+"up") || !delivery.FailedUntilChanged(err.Error()) {
t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, delivery.NotPinned)
}
}
func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 clause 5
_, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"}))
var r *Refused
if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), delivery.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") {
t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, delivery.RefusedKind)
}
}
func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) {
if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), RefusedPrefix) {
if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), delivery.RefusedKind) {
t.Errorf("a python bundle: error = %v, want a refusal", err)
}
p, err := PlanOf(manifestOf())
modules/mesh-delivery-builder/internal/build/r4e_test.go
+385
View File
@@ -0,0 +1,385 @@
package build
import (
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait
// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4).
func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3
r := newRig(t, goBundle, archiveA)
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != delivery.BuildDone {
t.Fatalf("build = %+v, want done", v)
}
if ok, missing := v.Referenced(); !ok {
t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts)
}
rev := r.st.revs[r.key]
r.bm.mu.Lock()
r.bm.calls = nil
r.bm.mu.Unlock()
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint
r.b.Wait()
if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev {
t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev)
}
}
func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2
r := newRig(t, goBundle)
r.bm.noRef["ace/tool"] = true
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "tool") {
t.Fatalf("build = %+v, want failed opening %q and naming the artifact", v, delivery.NotStored)
}
delete(r.bm.noRef, "ace/tool")
if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone {
t.Errorf("build = %+v, want a not-stored key built again as a new revision", v)
}
}
func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1, ADR 0498 clause 2
r := newRig(t, goBundle)
r.bm.fails["ace/build_go_build"] = delivery.NotStored + "tool: the artifact store answered 507 insufficient storage"
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") || !strings.Contains(v.Error, "ace") {
t.Fatalf("build = %+v, want failed opening %q with the store's words and the node", v, delivery.NotStored)
}
}
func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key
r := newRig(t, goBundle)
r.consider(t, "modules/m")
for _, c := range r.bm.callsOf("build_go_build") {
if c.body.(map[string]any)["publish"] != true {
t.Errorf("trunk build args = %+v, want publish", c.body)
}
}
c := newRig(t, goBundle)
trunkModule(t, c)
c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
c.b.Wait()
for _, call := range c.bm.callsOf("build_go_build") {
if _, ok := call.body.(map[string]any)["publish"]; ok {
t.Errorf("candidate build args = %+v, want no publish", call.body)
}
}
}
// putBuild writes a builder key's value as the store would hold it.
func putBuild(t *testing.T, st *memStore, key string, b delivery.BuilderBuild) uint64 {
t.Helper()
data, err := delivery.EncodeBuilderBuild(b)
if err != nil {
t.Fatal(err)
}
st.mu.Lock()
defer st.mu.Unlock()
return st.put(key, data)
}
func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { // B2, ADR 0498 clause 8
r := newRig(t, goBundle, archiveA)
rev := putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "built mesh-build-go on ace and novox",
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest}, {Name: "tool", Kind: "bundle", Digest: digest}}})
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev {
t.Fatalf("build = %+v at revision %d (was %d), want built again as a new revision", v, r.st.revs[r.key], rev)
}
if ok, missing := v.Referenced(); !ok {
t.Fatalf("the new revision names no reference for %v", missing)
}
rev = r.st.revs[r.key]
r.b.tried = map[string]bool{} // a builder started anew looks again
r.consider(t, "modules/m")
if r.st.revs[r.key] != rev {
t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev)
}
}
func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4
up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}
r := newRig(t, up)
r.b.Health = imageHave(t, "ace", "novox")
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
t.Fatalf("build = %+v, want done with the copy", v)
}
if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest {
t.Errorf("artifact = %+v, want the upstream's digest and the registry's reference", a)
}
copies := r.bm.callsOf("build_image_copy")
if len(copies) != 1 {
t.Fatalf("%d copies, want one on one node", len(copies))
}
if args := copies[0].body.(map[string]any); args["module"] != "m" || args["artifact"] != "up" || args["from"] != up.From || len(args) != 3 {
t.Errorf("copy args = %+v, want the module's name, the artifact and the pinned reference", args)
}
if n := len(r.bm.callsOf("build_image_fetch")) + len(r.bm.callsOf("build_image_build")); n != 0 {
t.Errorf("%d fetch or build calls for a copy, want none", n)
}
}
func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4
r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
r.b.Health = imageHave(t, "ace")
v := r.consider(t, "modules/m")
if ok, missing := v.Referenced(); !ok {
t.Fatalf("build = %+v, missing references %v", v, missing)
}
if f, b, c := len(r.bm.callsOf("build_image_fetch")), len(r.bm.callsOf("build_image_build")), len(r.bm.callsOf("build_image_copy")); f != 1 || b != 1 || c != 1 {
t.Errorf("fetch %d, build %d, copy %d; want one each", f, b, c)
}
}
func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2
r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
r.b.Health = imageHave(t, "ace")
r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500"
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) {
t.Fatalf("build = %+v, want failed 'not stored:' and built again", v)
}
delete(r.bm.fails, "ace/build_image_copy")
if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone {
t.Errorf("build = %+v, want done on the new revision", v)
}
}
var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"}
func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4
c := newRig(t, tsPackage)
c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, c)
c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
c.b.Wait()
v := valueOf(t, c.st, c.key)
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" {
t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v)
}
for _, call := range c.bm.callsOf("build_node_build") {
if _, ok := call.body.(map[string]any)["publish"]; ok {
t.Errorf("a candidate's package build was asked to publish: %+v", call.body)
}
}
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
v = r.consider(t, "modules/m")
if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" {
t.Fatalf("trunk build = %+v, want done naming the published version", v)
}
if n := len(r.bm.callsOf("build_node_build")); n == 0 {
t.Error("the package was not built by the Node build module")
}
}
func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B5, ADR 0498 clause 5
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
r.bm.fails["ace/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file"
r.bm.fails["novox/build_node_build"] = r.bm.fails["ace/build_node_build"]
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) || !strings.Contains(v.Error, "0.1.15") {
t.Fatalf("build = %+v, want failed opening %q naming the version", v, delivery.VersionTaken)
}
writes := r.st.writes
r.consider(t, "modules/m")
if r.st.writes != writes {
t.Errorf("a version-taken key was built again (%d writes, was %d)", r.st.writes, writes)
}
}
func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6
r := newRig(t, tsPackage, tsBundle)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
t.Fatalf("candidate build = %+v", v)
}
rev := r.st.revs[r.key]
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint
r.b.Wait()
v := valueOf(t, r.st, r.key)
if r.st.revs[r.key] <= rev {
t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev)
}
if ok, missing := v.Referenced(); !ok {
t.Fatalf("build = %+v, missing references %v", v, missing)
}
rev = r.st.revs[r.key]
r.b.tried = map[string]bool{}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if r.st.revs[r.key] != rev {
t.Error("a published package was built again: once only")
}
}
func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
for _, n := range []string{"ace", "novox"} {
r.bm.fails[n+"/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file"
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) {
t.Errorf("build = %+v, want failed opening %q", v, delivery.VersionTaken)
}
}
// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref.
func sdkModule(t *testing.T, r *rig, ref string) []byte {
t.Helper()
r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{tsPackage}}}
r.bm.packages["lib"] = ref
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "8a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: "src2:" + hex64('5'),
Manifest: mdigest('b'), Path: ".", Toolchains: agreedToolchains})
if err != nil {
t.Fatal(err)
}
return data
}
func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePublished(t *testing.T) { // B7, ADR 0500 clause 6
const ref = "@novox/mesh-sdk@0.1.15"
r := newRig(t, tsBundle)
for _, n := range []string{"ace", "novox"} {
r.bm.fails[n+"/build_node_fetch"] = waitsFor + ref + ": the lock names a version the registry does not hold"
}
v := r.consider(t, "modules/m")
if v.State != delivery.BuildQueued || v.Waits != ref || v.Error != "" {
t.Fatalf("build = %+v, want queued, waiting for %s, not failed", v, ref)
}
delete(r.bm.fails, "ace/build_node_fetch")
delete(r.bm.fails, "novox/build_node_fetch")
writes := r.st.writes
r.b.tried = map[string]bool{}
r.consider(t, "modules/m")
if r.st.writes != writes {
t.Fatalf("a key waiting for an unpublished package was claimed (%d writes, was %d)", r.st.writes, writes)
}
r.b.Module(context.Background(), "sdk", sdkModule(t, r, ref)) // the package's trunk key is built and published
r.b.Wait()
if v = valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Waits != "" {
t.Errorf("build = %+v, want built once the package it waits for is published", v)
}
}
func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *testing.T) { // B7: the scan at start
const ref = "@novox/mesh-sdk@0.1.15"
r := newRig(t, tsBundle)
putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildQueued, By: "novox.builder", Manifest: manifestD, Waits: ref, Step: "waits"})
sdkKey, _ := delivery.BuilderBuildKey("sdk", "src2:"+hex64('5'))
putBuild(t, r.st, sdkKey, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: mdigest('b'), Step: "built",
Artifacts: []delivery.BuiltArtifact{{Name: "lib", Kind: "package", Digest: digest, Reference: ref}}})
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
if err := r.b.Sweep(context.Background()); err != nil {
t.Fatal(err)
}
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
t.Errorf("build = %+v, want built: the package it waits for was published before the builder started", v)
}
}
func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *testing.T) { // the lead's interface (c), ADR 0499 clause 4
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')}
tree, err := delivery.NamedTree(manifestD, paths)
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".",
Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
r.b.Wait()
f := r.bm.callsOf("build_node_fetch")
if len(f) == 0 {
t.Fatal("no fetch")
}
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
if err != nil || req.Dir != "." || strings.Join(req.Paths, ",") != "package.json,src" || req.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" {
t.Errorf("fetch request = %+v, %v; want dir . and the kept paths, sorted", req, err)
}
r2 := newRig(t, goBundle)
r2.consider(t, "modules/m")
if a := r2.bm.callsOf("build_go_fetch")[0].body.(map[string]any); a["dir"] != nil || a["paths"] != nil {
t.Errorf("fetch args = %+v, want no dir or paths for a module naming none", a)
}
}
func hex40(c byte) string { return strings.Repeat(string(c), 40) }
func TestOneNodeBuildsEachFingerprint(t *testing.T) { // B9, ADR 0478 clause 4: "From R4e one node builds each fingerprint"
r := newRig(t, goBundle, archiveA, tsBundle) // ace and novox both qualify for both build modules
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || strings.Contains(v.Step, " and ") || strings.HasPrefix(v.Step, "one node (") {
t.Fatalf("build = %+v, want done on one node of each build module, unmarked", v)
}
if f, b := len(r.bm.callsOf("build_go_fetch")), len(r.bm.callsOf("build_go_build")); f != 1 || b != 2 {
t.Errorf("go fetches %d, builds %d; want one fetch and one build of each artifact, on one node", f, b)
}
if f := len(r.bm.callsOf("build_node_fetch")); f != 1 {
t.Errorf("node fetches %d, want one", f)
}
}
func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: the second-node retake is gone
r := newRig(t, goBundle)
putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD,
Step: "one node (mesh-build-go): built mesh-build-go on one node, novox",
Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/tool/blobs/" + digest}}})
writes := r.st.writes
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.NodesChanged(context.Background())
r.b.Wait()
if r.st.writes != writes || len(r.bm.calls) != 0 {
t.Errorf("a key done on one node was built again (%d writes, was %d; %d calls)", r.st.writes, writes, len(r.bm.calls))
}
}
func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths
r := newRig(t, tsPackage)
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
trunkModule(t, r)
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: headX}, Modules: []string{"m"},
Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
r.b.Candidate(context.Background(), "feat%2Fx", data)
r.b.Wait()
f := r.bm.callsOf("build_node_fetch")
if len(f) == 0 {
t.Fatal("no fetch")
}
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
}
}
modules/mesh-delivery-builder/internal/build/round4_test.go
+5 -85
View File
@@ -34,8 +34,8 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 {
t.Fatalf("build = %+v, want done with the archive and no toolchain named", v)
}
if n := len(r.bm.callsOf("build_go_build")); n != 2 {
t.Errorf("%d go builds, want one on each of two nodes", n)
if n := len(r.bm.callsOf("build_go_build")); n != 1 {
t.Errorf("%d go builds, want one, on one node (ADR 0478 clause 4)", n)
}
}
@@ -54,8 +54,8 @@ func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *te
r.b.Reconsider(context.Background())
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix) {
t.Errorf("build = %+v, want done on novox, the one node that qualifies, marked as one node", v)
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") {
t.Errorf("build = %+v, want done on novox, the one node that qualifies", v)
}
}
@@ -70,25 +70,6 @@ func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testi
}
}
// 2. The digests-differ failure opens with its prefix, and nothing builds it again.
func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testing.T) { // review round 4, 2
r := newRig(t, goBundle)
r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, "digests differ:") {
t.Fatalf("build = %+v, want a failure opening with exactly %q", v, "digests differ:")
}
calls, writes := len(r.bm.calls), r.st.writes
r.bm.digests = map[string]string{} // the nodes would now agree
r.b.Reconsider(context.Background())
r.b.Wait()
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if len(r.bm.calls) != calls || r.st.writes != writes {
t.Errorf("a failed-for-digests key was built again: %d calls (was %d), %d writes (was %d)", len(r.bm.calls), calls, r.st.writes, writes)
}
}
func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2
st := newMem()
seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"})
@@ -99,67 +80,6 @@ func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review
}
}
// 3. A done key built on one node is built again on a second node once one qualifies.
func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *testing.T) { // review round 4, 3
r := newRig(t, goBundle)
r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+buildrun.GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") {
t.Fatalf("build = %+v, want done on one node, marked", v)
}
if got := OneNodeModules(v); len(got) != 1 || got[0] != buildrun.GoBuildModule {
t.Errorf("OneNodeModules = %v", got)
}
rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build"))
// Nothing changes while one node qualifies: looking again builds nothing.
r.b.Reconsider(context.Background())
r.b.Wait()
if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_go_build")) != calls {
t.Fatal("a one-node key was built again with no second node")
}
// ace comes online: the key is built on ace and novox, as a new revision.
r.pres.Put("ace", presenceValue(t, true), false)
r.b.Reconsider(context.Background())
r.b.Wait()
v = valueOf(t, r.st, r.key)
if v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") {
t.Errorf("build = %+v, want done on ace and novox", v)
}
if r.st.revs[r.key] <= rev {
t.Errorf("revision %d, was %d: not a new revision", r.st.revs[r.key], rev)
}
builds := r.bm.callsOf("build_go_build")
if len(builds) != calls+2 || builds[len(builds)-2].node == builds[len(builds)-1].node {
t.Errorf("%d builds (was %d), want one on each of two nodes", len(builds), calls)
}
}
func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.T) { // review round 4, 3
r := newRig(t, goBundle)
r.pres.Put("ace", presenceValue(t, false), false)
r.consider(t, "modules/m")
r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
r.pres.Put("ace", presenceValue(t, true), false)
r.b.Reconsider(context.Background())
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, buildrun.DigestsDifferPrefix) {
t.Errorf("build = %+v, want failed with digests differ", v)
}
}
func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testing.T) { // review round 4, 3
r := newRig(t, goBundle)
r.pres.Put("ace", presenceValue(t, false), false)
first := r.consider(t, "modules/m")
r.bm.fails["ace/build_go_fetch"] = "ace is out of disk"
r.pres.Put("ace", presenceValue(t, true), false)
r.b.Reconsider(context.Background())
r.b.Wait()
if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) {
t.Errorf("build = %+v, want the first done build to stand: %+v", v, first)
}
}
// 4. A step never waits on a full node while another qualifying node has room.
func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4
p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}}
@@ -252,7 +172,7 @@ func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
r.b.Health = have
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on one node, novox") {
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") {
t.Fatalf("build = %+v, want done on novox alone", v)
}
for _, c := range r.bm.calls {
modules/mesh-delivery-builder/internal/build/run.go
+201 -117
View File
@@ -5,13 +5,14 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
@@ -41,8 +42,6 @@ type Builder struct {
}
// buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node.
Places *buildrun.Places
// Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4).
Nodes int
// Limit is how many builds run at once; four when 0.
Limit int
Logf func(format string, args ...any)
@@ -59,10 +58,9 @@ type Builder struct {
via map[string]via // by build key: a candidate's fetch (candidates.go)
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
tried map[string]bool // by build key: looked at since this process started
// epoch counts the changes of the nodes (online, healthy, a toolchain) seen since this process started; a done key
// whose second-node build failed is held back until it moves (issue 1664).
epoch uint64
secondAt map[string]uint64 // by build key: the epoch a second-node build of a one-node key failed at
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
// waiting for one is built once it is here (ADR 0500 clause 6).
published map[string]bool
}
type planned struct {
@@ -79,7 +77,7 @@ func (b *Builder) init() {
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.secondAt = map[string]uint64{}
b.published = map[string]bool{}
if b.Places == nil {
b.Places = &buildrun.Places{}
}
@@ -116,20 +114,14 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
}
// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or
// stopped being, or a toolchain moved. A done key built on one node whose second-node build failed is tried again only
// after such a change, not at every modules value (issue 1664).
func (b *Builder) NodesChanged(ctx context.Context) {
b.init()
b.mu.Lock()
b.epoch++
b.mu.Unlock()
b.Reconsider(ctx)
}
// stopped being, or a toolchain moved, so a build that waited for a node may go on.
func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) }
// Sweep closes, at start, every key left building for a fingerprint its module no longer has (issue 1660, ADR 0478
// clause 6): the key can never be built, since nothing asks for it, and would stay building for ever. It fails the key
// naming who left it. A key of the fingerprint the module has now is not touched: the build takes it over. Call it
// once the modules bucket has been read, so every module is known.
// Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key
// waiting for one is then looked at again, ADR 0500 clause 6), and closes every key left building for a fingerprint its
// module no longer has (issue 1660, ADR 0478 clause 6): the key can never be built, since nothing asks for it, and would
// stay building for ever. It fails the key naming who left it. A key of the fingerprint the module has now is not
// touched: the build takes it over. Call it once the modules bucket has been read, so every module is known.
func (b *Builder) Sweep(ctx context.Context) error {
b.init()
keys, err := b.Claimer.Store.Keys(ctx)
@@ -146,16 +138,18 @@ func (b *Builder) Sweep(ctx context.Context) error {
}
}
b.mu.Unlock()
newly := false
for _, key := range keys {
if current[key] {
continue
}
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil || !found {
continue
}
cur, err := delivery.DecodeBuilderBuild(value)
if err != nil || cur.State != delivery.BuildBuilding {
if err != nil {
continue
}
newly = b.notePublished(cur) || newly
if current[key] || cur.State != delivery.BuildBuilding {
continue
}
closed := delivery.BuilderBuild{State: delivery.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step,
@@ -169,9 +163,36 @@ func (b *Builder) Sweep(ctx context.Context) error {
}
b.logf("%s: closed, left building by %s for a fingerprint the module no longer has", key, cur.By)
}
if newly {
// A key read before the scan may wait for a package the scan found published (ADR 0500 clause 6).
b.Reconsider(ctx)
}
return nil
}
// notePublished keeps the package versions a done build names as references, and says whether one was new.
func (b *Builder) notePublished(v delivery.BuilderBuild) bool {
if v.State != delivery.BuildDone {
return false
}
b.mu.Lock()
defer b.mu.Unlock()
newly := false
for _, a := range v.Artifacts {
if a.Kind == catalogue.ArtifactPackage && a.Reference != "" && !b.published[a.Reference] {
b.published[a.Reference] = true
newly = true
}
}
return newly
}
func (b *Builder) isPublished(ref string) bool {
b.mu.Lock()
defer b.mu.Unlock()
return b.published[ref]
}
// Forget takes a module out (its key was deleted).
func (b *Builder) Forget(key string) {
b.init()
@@ -275,10 +296,8 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
plan, planErr, read := b.planOf(ctx, m.Manifest)
if read && planErr == nil && len(plan.Tasks) == 0 {
return // nothing in the manifest for a builder to build
}
if planErr == nil {
nothing := read && planErr == nil && len(plan.Tasks) == 0
if planErr == nil && !nothing {
if why := b.stale(m, plan); why != "" {
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
@@ -291,14 +310,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
trunk := b.trunkFP[key]
first := !b.tried[key]
b.tried[key] = true
epoch := b.epoch
failedAt, failedBefore := b.secondAt[key]
b.mu.Unlock()
moreNodes := func(modules []string) bool {
// A second build that failed is not tried again until the nodes change (issue 1664).
return (!failedBefore || epoch > failedAt) && b.moreNodes(modules)
}
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: moreNodes, ImageBuilder: b.imageBuilderRuns, Trunk: trunk})
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished})
if err != nil {
b.logf("%s: %v", module, err)
return
@@ -306,22 +319,28 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
if cl == nil {
return
}
// A done key taken again builds only what is missing: the build modules that ran on one node, and the image when it was
// built as a candidate's; the other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8).
// A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the
// other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6).
var prior *delivery.BuilderBuild
var only map[string]bool
publishing := false
if cl.Build.State == delivery.BuildDone {
prior, only = &cl.Build, map[string]bool{}
for _, bm := range OneNodeModules(cl.Build) {
only[bm] = true
}
if trunk && Unpublished(cl.Build) {
only[ImageBuildModule] = true
publishing = true
if trunk {
for _, bm := range publishable(cl.Build, b.imageBuilderRuns) {
only[bm] = true
publishing = true
}
}
}
result := b.build(ctx, module, m, plan, planErr, prior, only)
var result delivery.BuilderBuild
if nothing {
// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"}
} else {
result = b.build(ctx, module, m, plan, planErr, prior, only)
}
if ctx.Err() != nil {
return
}
@@ -331,17 +350,9 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
}
result.By, result.Manifest = b.Claimer.By, m.Manifest
if publishing && result.State == delivery.BuildFailed {
// The trunk's image was built only as a candidate's and its push failed: the key says so, so the builds view, the
// The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the
// checker and the failed-key retake see it, and a look does not build the image again for nothing.
result.Error = "publishing the image of the merged commit failed: " + result.Error
} else if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, buildrun.DigestsDifferPrefix) {
// A done key built again on a second node keeps its done value when the new build fails for another reason than
// the digests: the first build stands, and a later change of the nodes looks again.
b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error)
b.mu.Lock()
b.secondAt[cl.Key] = epoch
b.mu.Unlock()
return
result.Error = keyError([]string{"publishing what the merged commit built failed: " + result.Error})
}
if err := b.Claimer.Finish(ctx, cl, result); err != nil {
b.logf("%s: the build could not be written: %v", cl.Key, err)
@@ -350,6 +361,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
if result.State == delivery.BuildFailed {
b.logf("%s: failed: %s", cl.Key, result.Error)
}
if result.State == delivery.BuildQueued {
b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits)
}
if b.notePublished(result) {
// A key may wait for the package this build published (ADR 0500 clause 6).
b.Reconsider(ctx)
}
}
// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
@@ -396,7 +414,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
repository := delivery.RepositoryName(m.Repository)
var lanes []*lane
used := map[string]delivery.ToolchainUsed{}
var steps, oneNode []string
var steps []string
// Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up.
giveUp := func(nodes ...string) {
for _, n := range nodes {
@@ -429,7 +447,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
}
nodes = ns
} else if plan.NeedsToolchain(bm) {
ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, b.Nodes)
ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, 1)
if err != nil {
giveUpLanes()
return failed("%s", err)
@@ -443,23 +461,15 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
nodes = ns
} else {
// An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies.
ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes)
ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, 1)
if err != nil {
giveUpLanes()
return failed("%s", err)
}
nodes = ns
}
if bm == ImageBuildModule {
steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0]))
} else if len(nodes) == 1 {
oneNode = append(oneNode, bm)
steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0]))
} else {
sorted := append([]string(nil), nodes...)
sort.Strings(sorted)
steps = append(steps, fmt.Sprintf("%s on %s", bm, strings.Join(sorted, " and ")))
}
// One node builds each fingerprint (ADR 0478 clause 4, from R4e).
steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0]))
var tasks []Task
for _, t := range plan.Tasks {
if t.Module == bm {
@@ -483,7 +493,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
}
}
if len(errs) > 0 {
return failed("%s", strings.Join(errs, "; "))
if ref := waitsOf(errs); ref != "" {
// The module's lock names a version of a package the registry does not hold yet: the key waits for it, queued, and
// is not failed (ADR 0500 clause 6).
return delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref, Step: "waits for the package " + ref + " to be published"}
}
return failed("%s", keyError(errs))
}
var artifacts []delivery.BuiltArtifact
keptModule := map[string]bool{}
@@ -515,7 +530,16 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
}
built := delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d}
for _, l := range lanes {
if r := l.references[t.Artifact]; r != "" && l.module == t.Module {
if l.module != t.Module {
continue
}
r := l.references[t.Artifact]
if r == "" && stored(t.Kind) {
// A bundle, an archive or an upstream copy is done only when its push was answered with where it is (ADR 0498
// clause 1); a candidate's included (clause 3).
return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", delivery.NotStored, t.Artifact, l.node)
}
if r != "" {
built.Reference = r
}
}
@@ -530,14 +554,64 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
}
}
step := "built " + strings.Join(steps, "; ")
if len(oneNode) > 0 {
// Marked so that Claim builds the key again once a second node qualifies (ADR 0478 clause 4).
step = OneNodePrefix + strings.Join(oneNode, ", ") + "): " + step
}
if len(used) == 0 {
used = nil
}
return delivery.BuilderBuild{State: delivery.BuildDone, Step: step, Artifacts: artifacts, Toolchains: used}
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used}
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
// A reference the build module answered that does not name the artifact and its digest is no reference.
return failed("%s%v", delivery.NotStored, err)
}
return done
}
// stored is whether an artifact of this kind is in the artifact store once built, a candidate's too, so its done key
// names its reference (ADR 0498 clauses 1, 3 and 4); an image or a package names one only when built for the trunk.
func stored(kind string) bool {
return kind == catalogue.ArtifactBundle || kind == catalogue.ArtifactArchive || kind == catalogue.ArtifactUpstream
}
// waitsFor opens the reason of a fetch step whose module's lock names a package version the registry does not hold, then
// `<name>@<version>` (the Node build module's, ADR 0500 clause 6).
const waitsFor = delivery.WaitsFor
// waitsOf is the package version a lane's error says the module waits for, or "" when none does or what it names is not
// `<name>@<version>`.
func waitsOf(errs []string) string {
for _, e := range errs {
at := strings.Index(e, waitsFor)
if at < 0 {
continue
}
ref, _, _ := strings.Cut(e[at+len(waitsFor):], " ")
ref = strings.TrimRight(ref, ":;,")
if _, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref}); err == nil {
return ref
}
}
return ""
}
// keyPrefixes are the errors a build module's step may open its reason with that the key's error must open with too, so
// the claim reads them (delivery.FailedUntilChanged), first the ones not built again.
var keyPrefixes = []string{delivery.NotPinned, delivery.VersionTaken, delivery.RefusedKind, delivery.NotStored}
// keyError is the error of a key from its lanes' errors: when one names a prefix of keyPrefixes inside the step's words
// (`<tool> on <node>: the step <key> failed: not stored: ...`), the key's error opens with that prefix, so a reader of the
// key need not search for it.
func keyError(errs []string) string {
for _, prefix := range keyPrefixes {
for i, e := range errs {
if strings.HasPrefix(e, prefix) {
return strings.Join(errs, "; ")
}
if at := strings.Index(e, prefix); at >= 0 {
rest := append([]string{prefix + e[:at] + e[at+len(prefix):]}, append(append([]string(nil), errs[:i]...), errs[i+1:]...)...)
return strings.Join(rest, "; ")
}
}
}
return strings.Join(errs, "; ")
}
func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) }
@@ -580,24 +654,6 @@ func (b *Builder) imageBuilderRuns() bool {
return false
}
// moreNodes is whether any of these build modules has two nodes that qualify now.
func (b *Builder) moreNodes(modules []string) bool {
for _, bm := range modules {
seen := map[string]bool{}
n := 0
for _, e := range b.Toolchains.Entries(buildrun.LanguageOf[bm]) {
if !seen[e.Node] && b.qualifies(e.Node, bm) {
seen[e.Node] = true
n++
}
}
if n >= 2 {
return true
}
}
return false
}
// step runs one step of a build module; the lane that runs it holds its node's place.
func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) {
return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args)
@@ -616,6 +672,8 @@ type buildResult struct {
}
// runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time.
// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches
// nothing.
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) {
release, err := b.Places.Acquire(ctx, l.node)
if err != nil {
@@ -623,57 +681,83 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
return
}
defer release()
fetchArgs := map[string]any{"repository": repository, "commit": m.Commit}
req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit}
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
b.mu.Lock()
v, candidate := b.via[key]
b.mu.Unlock()
if candidate {
fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base
req.Ref, req.Base = v.Ref, v.Base
}
}
raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs)
if err != nil {
l.err = err
return
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
// A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from
// the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e).
req.Dir = m.Path
for p := range m.Inputs2.Paths {
req.Paths = append(req.Paths, p)
}
sort.Strings(req.Paths)
}
fetchArgs := req.Args()
candidate := req.Base != ""
var fetched fetchResult
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
return
}
// A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at
// the trunk, and building that would store the trunk's build under the candidate's fingerprint.
_, candidate := fetchArgs["base"]
if candidate {
if len(fetched.Conflicts) > 0 {
l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", "))
if needsFetch(l.tasks) {
raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs)
if err != nil {
l.err = err
return
}
if fetched.Merge == "" {
l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw)
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
return
}
// A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at
// the trunk, and building that would store the trunk's build under the candidate's fingerprint.
if candidate {
if len(fetched.Conflicts) > 0 {
l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", "))
return
}
if fetched.Merge == "" {
l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw)
return
}
}
}
for _, t := range l.tasks {
args := map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact}
if t.Kind == catalogue.ArtifactImage && !candidate {
// Only a trunk key pushes the image to the registry; a candidate's build never does (ADR 0487 clause 12).
tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact}
if t.Kind == catalogue.ArtifactUpstream {
// The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin.
tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From}
} else if !candidate {
// Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12,
// ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3).
args["publish"] = true
}
raw, err := b.step(ctx, l.module, l.node, "build", args)
raw, err := b.step(ctx, l.module, l.node, tool, args)
if err != nil {
l.err = err
return
}
var built buildResult
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw)
l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw)
return
}
l.digests[t.Artifact] = built.Digest
if built.Reference != "" && !candidate {
if built.Reference != "" && (stored(t.Kind) || !candidate) {
l.references[t.Artifact] = built.Reference
}
}
}
// needsFetch is whether any of the tasks builds from the module's source: every one but an upstream copy.
func needsFetch(tasks []Task) bool {
for _, t := range tasks {
if t.Kind != catalogue.ArtifactUpstream {
return true
}
}
return false
}
modules/mesh-delivery-builder/internal/build/run_test.go
+65 -84
View File
@@ -23,6 +23,10 @@ type buildModules struct {
steps *fakeSteps
digests map[string]string // `<node>/<artifact>` -> digest; the default is `digest`
fails map[string]string // `<node>/<tool>` -> the reason the step fails with
// noRef names `<node>/<artifact>` whose bundle or archive build answers no reference (its push not answered).
noRef map[string]bool
// packages are the package artifacts by name, each with the `<name>@<version>` its trunk build publishes (ADR 0498 clause 5).
packages map[string]string
// hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives.
// conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict.
conflicts string
@@ -32,7 +36,7 @@ type buildModules struct {
}
func newBuildModules() *buildModules {
return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}}
return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}}
}
func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) {
@@ -67,8 +71,18 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro
if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok {
d = v
}
if tool == "build_image_build" && args["publish"] == true {
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/img@` + d + `"}`)
if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4)
step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`)
} else if pkg, ok := b.packages[fmt.Sprint(args["artifact"])]; ok { // a package names its version only when published
step.Result = json.RawMessage(`{"digest":"` + d + `"}`)
if args["publish"] == true {
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"` + pkg + `"}`)
}
} else if tool == "build_image_build" && args["publish"] == true {
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`)
} else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] {
// A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1).
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`)
} else {
step.Result = json.RawMessage(`{"digest":"` + d + `"}`)
}
@@ -125,7 +139,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
}
r.b = &Builder{
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second},
Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf,
Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf,
}
var err error
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
@@ -166,15 +180,15 @@ var (
manifestD = mdigest('a')
)
func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
r := newRig(t, goBundle, archiveA, tsBundle)
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v)
}
for i, want := range []string{"files", "tool", "web"} {
if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "" {
t.Errorf("artifact %d = %+v, want %s with the digest and no reference (ADR 0478 clause 6)", i, a, want)
if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "artifact-store://m/"+want+"/blobs/"+digest {
t.Errorf("artifact %d = %+v, want %s with the digest and its artifact-store reference (ADR 0498 clause 1)", i, a, want)
}
}
if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" {
@@ -186,15 +200,15 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test
if got := v.Toolchains["typescript"]; got.Version != "tsc5" {
t.Errorf("toolchains = %+v", v.Toolchains)
}
// Two fetches per build module, one on each node; every build names the module's directory.
if n := len(r.bm.callsOf("build_go_fetch")); n != 2 {
t.Errorf("%d go fetches, want one per node", n)
// One fetch per build module, on one node (ADR 0478 clause 4); every build names the module's directory.
if n := len(r.bm.callsOf("build_go_fetch")); n != 1 {
t.Errorf("%d go fetches, want one", n)
}
if n := len(r.bm.callsOf("build_go_build")); n != 4 { // tool and files, each on two nodes
t.Errorf("%d go builds, want 4", n)
if n := len(r.bm.callsOf("build_go_build")); n != 2 { // tool and files
t.Errorf("%d go builds, want 2", n)
}
if n := len(r.bm.callsOf("build_node_build")); n != 2 {
t.Errorf("%d node builds, want 2", n)
if n := len(r.bm.callsOf("build_node_build")); n != 1 {
t.Errorf("%d node builds, want 1", n)
}
c := r.bm.callsOf("build_go_build")[0]
if args := c.body.(map[string]any); args["module"] != "modules/m" || !strings.HasPrefix(fmt.Sprint(args["workspace"]), "w-") {
@@ -206,21 +220,12 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test
}
}
func TestDifferingDigestsFailTheBuildNamingBothNodes(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
r := newRig(t, goBundle)
r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "novox") || !strings.Contains(v.Error, digest) || !strings.Contains(v.Error, hex64('9')) {
t.Errorf("build = %+v, want failed naming both nodes and both digests", v)
}
}
func TestOneQualifyingNodeBuildsOnceAndTheKeySaysSo(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
func TestOneQualifyingNodeBuildsOnce(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
r := newRig(t, goBundle)
r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true)
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "one node") || !strings.Contains(v.Step, "novox") {
t.Errorf("build = %+v, want done with its step saying one node, novox", v)
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") {
t.Errorf("build = %+v, want done with its step saying novox", v)
}
if n := len(r.bm.callsOf("build_go_build")); n != 1 {
t.Errorf("%d builds, want one", n)
@@ -263,9 +268,6 @@ func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { /
if v.Artifacts[0].Reference == "" {
t.Errorf("a trunk image has no reference: %+v", v.Artifacts[0])
}
if strings.HasPrefix(v.Step, OneNodePrefix) || len(OneNodeModules(v)) != 0 {
t.Errorf("step = %q, want no one-node mark: an image is never built again on a second node", v.Step)
}
if n := len(r.bm.callsOf("build_image_fetch")); n != 1 {
t.Errorf("%d image fetches, want one", n)
}
@@ -286,22 +288,19 @@ func TestAnImageIsBuiltOnlyWhereMeshBuildImageIsHealthy(t *testing.T) { // 1705a
}
}
func TestABundleAndAnImageOfOneModuleBuildOnTwoNodesAndOne(t *testing.T) { // 1705a
func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
r := newRig(t, goBundle, imageArt)
r.b.Health = imageHave(t, "ace", "novox")
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || len(v.Artifacts) != 2 {
t.Fatalf("build = %+v", v)
}
if n := len(r.bm.callsOf("build_go_build")); n != 2 {
t.Errorf("%d go builds, want 2", n)
if n := len(r.bm.callsOf("build_go_build")); n != 1 {
t.Errorf("%d go builds, want 1 (one node per fingerprint, ADR 0478 clause 4)", n)
}
if n := len(r.bm.callsOf("build_image_build")); n != 1 {
t.Errorf("%d image builds, want 1", n)
}
if modules := OneNodeModules(v); len(modules) != 0 {
t.Errorf("one-node modules = %v, want none", modules)
}
}
func TestWithNoImageNodeTheKeyFailsNamingTheProvisionAndIsNotRetakenUntilANodeReportsTheModule(t *testing.T) { // 1705b, lead's decision
@@ -357,19 +356,19 @@ func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b
}
}
func TestAnUpstreamArtifactIsStillRefusedKeyNeverBuiltAgain(t *testing.T) { // 1607/AC2, ADR 0478 clause 2
r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream"})
func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4
r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"})
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, RefusedPrefix) || !strings.Contains(v.Error, "upstream") || !strings.Contains(v.Error, "build-agent") {
t.Fatalf("build = %+v, want failed 'refused kind: upstream' naming build-agent", v)
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotPinned) || !strings.Contains(v.Error, "up") {
t.Fatalf("build = %+v, want failed 'not pinned:' naming the artifact", v)
}
if len(r.bm.calls) != 0 {
t.Errorf("%d calls, want none for a refused kind", len(r.bm.calls))
t.Errorf("%d calls, want none for an unpinned upstream", len(r.bm.calls))
}
writes := r.st.writes
r.consider(t, "modules/m")
if r.st.writes != writes {
t.Errorf("a refused key was written again (%d writes, was %d)", r.st.writes, writes)
t.Errorf("an unpinned upstream's key was written again (%d writes, was %d)", r.st.writes, writes)
}
}
@@ -390,14 +389,33 @@ func TestAnUnreadableManifestFailsTheBuild(t *testing.T) {
}
}
func TestAModuleWithNoFingerprintOrNoArtifactIsNotBuilt(t *testing.T) { // 1607/AC2
func TestAModuleWithNoFingerprintIsNotBuilt(t *testing.T) { // 1607/AC2
r := newRig(t)
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"})
r.b.Module(context.Background(), "n", data)
r.b.Wait()
if len(r.st.vals) != 0 {
t.Errorf("keys %v were written: a module with no new-form fingerprint or no artifact has nothing to build", r.st.vals)
t.Errorf("keys %v were written: a module with no new-form fingerprint has nothing to build", r.st.vals)
}
}
func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testing.T) { // issue 1613, ADR 0478 clause 1
for name, manifest := range map[string]catalogue.Manifest{"no build": {Module: "m"}, "no artifact": {Module: "m", Build: &catalogue.Build{}}} {
r := newRig(t)
r.mf.byDigest[mdigest('a')] = manifest
v := r.consider(t, "") // a module with nothing to build needs no directory
if v.State != delivery.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" {
t.Errorf("%s: build = %+v, want done with no artifact", name, v)
}
if len(r.bm.calls) != 0 {
t.Errorf("%s: %d build module calls, want none", name, len(r.bm.calls))
}
writes := r.st.writes
r.b.tried = map[string]bool{}
r.consider(t, "")
if r.st.writes != writes {
t.Errorf("%s: the done key was written again", name)
}
}
}
@@ -421,7 +439,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *
// whose compare-and-set comes first, and the other's write is refused, never overwriting it.
func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4
r := newRig(t, goBundle)
other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf}
other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf}
value := moduleValue(t, "modules/m")
var wg sync.WaitGroup
for _, b := range []*Builder{r.b, other} {
@@ -432,8 +450,8 @@ func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/A
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
t.Errorf("build = %+v, want one done key", v)
}
if n := len(r.bm.callsOf("build_go_build")); n < 2 || n%2 != 0 {
t.Errorf("%d builds, want two for each instance that built", n)
if n := len(r.bm.callsOf("build_go_build")); n < 1 {
t.Errorf("%d builds, want one for each instance that built", n)
}
}
@@ -570,43 +588,6 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi
}
}
func TestASecondGoNodeDoesNotBuildOrPushTheImageAgain(t *testing.T) { // review r1, mesh-delivery 2; ADR 0487 clause 8
r := newRig(t, goBundle, imageArt)
r.b.Health = imageHave(t, "ace", "novox")
r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies for Go (and the image)
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || len(OneNodeModules(v)) != 1 {
t.Fatalf("build = %+v, want done with the Go bundle on one node", v)
}
var image delivery.BuiltArtifact
for _, a := range v.Artifacts {
if a.Kind == "image" {
image = a
}
}
if image.Reference == "" {
t.Fatalf("artifacts = %+v, want a published image", v.Artifacts)
}
r.pres.Put("ace", presenceValue(t, true), false)
r.b.Reconsider(context.Background())
r.b.Wait()
v = valueOf(t, r.st, r.key)
if len(OneNodeModules(v)) != 0 || len(v.Artifacts) != 2 {
t.Fatalf("build = %+v, want done on two Go nodes", v)
}
if n := len(r.bm.callsOf("build_image_build")); n != 1 {
t.Errorf("%d image builds, want the one of the first build only", n)
}
for _, a := range v.Artifacts {
if a.Kind == "image" && a != image {
t.Errorf("image = %+v, was %+v: the done key was rewritten with another image", a, image)
}
}
if n := len(r.bm.callsOf("build_go_build")); n != 3 { // one, then two
t.Errorf("%d go builds, want 3", n)
}
}
func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5
r := newRig(t, imageArt)
r.b.Health = imageHave(t, "ace")
modules/mesh-delivery-builder/manifest_test.go
+2 -2
View File
@@ -90,8 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node)", m.Invokes)
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes)
}
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
modules/mesh-delivery-builder/module.json
+1
View File
@@ -25,6 +25,7 @@
"mesh-build-go.build_go_build",
"mesh-build-go.build_go_fetch",
"mesh-build-image.build_image_build",
"mesh-build-image.build_image_copy",
"mesh-build-image.build_image_fetch",
"mesh-build-node.build_node_build",
"mesh-build-node.build_node_fetch"
modules/mesh-delivery-checker/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-checker/go.sum
+2 -4
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-checker/internal/check/preview.go
+2 -2
View File
@@ -14,7 +14,7 @@ var digest64 = regexp.MustCompile(`^[0-9a-f]{64}$`)
// PreviewOf is what the fetch steps said of a pull request's head merged onto its trunk head, as the value of a previews
// key (ADR 0484 clause 3): the merge commit and, for each module directory the fetches named with a manifest, its tree, the
// digest of its manifest and the hashes of its lock files. The fetches of the repository's build modules merge the same
// digest of its manifest, the hashes of its lock files and its named paths' objects (ADR 0499 clause 2). The fetches of the repository's build modules merge the same
// trees; the first one's merge commit is kept. A module directory with no tree or no manifest digest is left out, so the
// indexer gives its module no fingerprint and says why. A head that does not merge has a preview of the files in conflict
// alone, so the indexer says the candidate's modules have no fingerprint because it does not merge (issue 1445). It
@@ -43,7 +43,7 @@ func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) {
if t.Tree == "" || !digest64.MatchString(t.Manifest) {
continue
}
mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks}
mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks, Paths: t.Paths}
}
}
if merge == "" {
modules/mesh-delivery-checker/internal/check/preview_named_test.go
+23
View File
@@ -0,0 +1,23 @@
package check
import (
"encoding/json"
"reflect"
"testing"
)
// I8, ADR 0499 clause 2: the preview carries each named path's object a fetch's module trees answer (mesh-sdk
// buildstep.ModuleTree's paths), so the indexer computes the candidate's narrowed tree from them by the one function.
func TestAPreviewCarriesTheNamedPathsObjectsOfAFetchsModuleTrees(t *testing.T) {
var f Fetched
// As buildstep.ModuleTrees answers a module naming its paths: the named tree and the objects.
raw := `{"workspace":"w","merge":"6666666666666666666666666666666666666666","trees":{".":{"tree":"named:aa","manifest":"` +
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + `","paths":{"src":"1111111111111111111111111111111111111111"}}}}`
if err := json.Unmarshal([]byte(raw), &f); err != nil {
t.Fatal(err)
}
pv, ok := PreviewOf(Pull{Repository: "novox/mesh-sdk", Head: "h", Trunk: "t"}, []Fetched{f})
if !ok || !reflect.DeepEqual(pv.Modules["."].Paths, map[string]string{"src": "1111111111111111111111111111111111111111"}) || pv.Modules["."].Tree != "named:aa" {
t.Errorf("preview = %+v, want the module's named tree and its objects", pv)
}
}
modules/mesh-delivery-checker/internal/check/results.go
+5 -2
View File
@@ -10,12 +10,15 @@ import (
// cmd/mesh-build-node check.go TestResult, fetch.go FetchResult) with the fields the fetch step adds for a merge (ADR 0484
// clause 3). Only the fields the checker reads are named; the rest is ignored.
// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest and the
// hashes of its lock files by name. A bare string is the tree alone.
// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest, the
// hashes of its lock files by name and the objects of its named paths. A bare string is the tree alone.
type MergeTree struct {
Tree string `json:"tree"`
Manifest string `json:"manifest,omitempty"`
Locks map[string]string `json:"locks,omitempty"`
// Paths are, of a module whose every artifact names its paths, each named path's object; Tree is then their named
// tree (mesh-sdk buildstep.ModuleTree, novox/hq ADR 0499 clause 2).
Paths map[string]string `json:"paths,omitempty"`
}
// UnmarshalJSON takes a string as the tree alone, or an object.
modules/mesh-delivery-forge-bridge/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-forge-bridge/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go
+36
View File
@@ -0,0 +1,36 @@
package bridge
import (
"os"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a
// module whose artifacts name their paths (mesh-sdk's module.json at 9cb4d59) gives a message that carries the manifest as
// the forge answered it and no named path's object: the indexer asks the forge for those itself.
func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) {
raw, err := os.ReadFile("testdata/mesh-sdk-9cb4d59.module.json")
if err != nil {
t.Fatal(err)
}
files := func(owner, repo, path, ref string) ([]byte, error) {
if path == "module.json" {
return raw, nil
}
return nil, ErrNotFound
}
var calls []call
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}), treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), files)
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
if len(f.Paths) != 0 || f.Manifests[""] != string(raw) || len(calls) != 1 {
t.Errorf("message paths %v, manifest carried %v, trees asked %d times; want no paths, the manifest, one ask", f.Paths, f.Manifests[""] == string(raw), len(calls))
}
}
modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json
Vendored
+16
View File
@@ -0,0 +1,16 @@
{
"module": "mesh-sdk",
"version": "1",
"slug": "sdk",
"build": {
"artifacts": [
{
"name": "lib",
"kind": "package",
"language": "typescript",
"paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"]
}
]
},
"resources": []
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
+52 -14
View File
@@ -9,6 +9,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -183,30 +184,24 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b
// TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged
// message on FORGE makes modules.<m> within 5 s, the fingerprint of the merge's tree and the recorded inputs.
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
// Inputs and record in the controller's layout: SourceInputs under its Go field names (mesh-sdk
// go/delivery/testdata/build-record-build-agent.json, captured from the controller's encoder).
const inputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851)
// The bucket holds only build-agent's record, the one the indexer reads until R4g (the builder's own input).
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
rec := `{"id":"build-1","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
if _, err := builds.Put(ctx, delivery.BuildKey("a", "build-1"), []byte(rec)); err != nil {
t.Fatal(err)
}
if _, err := builds.Put(ctx, delivery.NewestKey("a"), []byte("build-1")); err != nil {
rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`
if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil {
t.Fatal(err)
}
})
const merge = "89abcdef0123456789abcdef0123456789abcdef"
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}})
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged", data); err != nil {
t.Fatal(err)
}
want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
var m delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "a"); err == nil {
@@ -215,8 +210,8 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
}
}
}
if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" { // 1606/AC2
t.Errorf("module = %+v within 5 s, want fp %s and path modules/a", m, want)
if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.FP != "" || m.None != "" { // 1606/AC2
t.Errorf("module = %+v within 5 s, want path modules/a, its manifest, and no value of today's form", m)
}
}
@@ -440,3 +435,46 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
t.Errorf("none2 = %q within 5 s of the preview, want %q (computed from the preview)", got, delivery.ToolchainNoReport)
}
}
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form
// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature).
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
if err != nil {
t.Fatal(err)
}
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
merged := func(merge, root string) delivery.Module {
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
t.Fatal(err)
}
var m delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
if m, err = delivery.DecodeModule(e.Value()); err == nil && m.Commit == merge {
return m
}
}
}
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
return m
}
first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2)
}
if first.FP2 != second.FP2 || first.None2 != second.None2 {
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
+13 -1
View File
@@ -25,6 +25,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
)
@@ -283,7 +284,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
errs := make(chan error, 1)
go func() { errs <- follow(ctx, w, each) }()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf}
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
account, err := process.ReadAccount(path)
if err != nil {
return err
}
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer calls.Close()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
Named: namedVia(calls.Call)}
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go
+48
View File
@@ -0,0 +1,48 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each
// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk
// delivery.Forge.Paths; novox/hq ADR 0499 clause 3).
const commitTreesTool = "gitea.gitea_commit_trees"
// toolCaller calls a module's tool, `<module>.<tool>`, as the indexer's own account (mesh-sdk process.Conn.Call).
type toolCaller func(key string, body any) (json.RawMessage, error)
// namedVia asks the forge for the objects of the named paths through call. An answer without the object of every path
// asked is an error naming it.
func namedVia(call toolCaller) func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
return func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
if !ok {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths})
if err != nil {
return nil, err
}
var a struct {
Paths map[string]map[string]string `json:"paths"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err)
}
for dir, want := range paths {
for _, p := range want {
if a.Paths[dir][p] == "" {
return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir)
}
}
}
return a.Paths, nil
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go
+31
View File
@@ -0,0 +1,31 @@
package main
import (
"context"
"encoding/json"
"testing"
)
func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3
var asked map[string]any
call := func(key string, body any) (json.RawMessage, error) {
if key != commitTreesTool {
t.Errorf("called %s", key)
}
asked = body.(map[string]any)
return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil
}
got, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}})
if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" {
t.Fatalf("objects = %v, %v", got, err)
}
if asked["owner"] != "novox" || asked["repo"] != "mesh-sdk" || asked["sha"] != "s" || asked["paths"] == nil {
t.Errorf("asked %v, want the owner, repo, commit and paths", asked)
}
call = func(string, any) (json.RawMessage, error) {
return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil
}
if _, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil {
t.Error("an answer missing a path asked was taken")
}
}
modules/mesh-delivery-indexer/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-indexer/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-indexer/internal/index/builds.go
+33 -98
View File
@@ -1,31 +1,24 @@
// Package index keeps the candidate groups (novox/hq ADR 0437 §5) the forge bridge's pull messages make.
package index
import (
"encoding/json"
"path"
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Builds holds, from the controller's builds bucket (mesh-controller_builds), which module a directory of a
// repository is: the module of the newest build record naming that repository and path (BuildRecord.Path). Safe for use
// from a watch and from the message handler at once.
// Builds holds the controller's build records (mesh-controller_builds) for the two things the indexer still reads from
// them until release R4g, when build-agent goes (the lead's decision of R4e, 2026-10-22):
// - the Builder input of the builder's own module, which build-agent builds (TrunkOf; novox/hq ADR 0453 clause 5,
// issue 1727);
// - the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no fingerprint.
//
// It lists no module and maps no directory to a module: the indexer's modules are those its own messages and bucket name
// (ADR 0501 clause 6, issue 1851). Safe for use from a watch and from the message handler at once.
type Builds struct {
mu sync.Mutex
records map[string]delivery.BuildRecord // by bucket key
byDir map[string]delivery.BuildRecord // by repository and path; nil when stale
mods map[string]*moduleBuilds // by module; nil when stale
}
// moduleBuilds is what the builds say of one module: its newest record of any kind, and its newest successful trunk
// build that recorded its inputs.
type moduleBuilds struct {
any delivery.BuildRecord
trunk *delivery.BuildRecord
}
// NewBuilds is an empty Builds.
@@ -44,101 +37,19 @@ func (b *Builds) Put(key string, value []byte) {
b.mu.Lock()
defer b.mu.Unlock()
b.records[key] = r
b.byDir, b.mods = nil, nil
}
// Delete takes a key out.
func (b *Builds) Delete(key string) {
b.mu.Lock()
defer b.mu.Unlock()
if _, ok := b.records[key]; ok {
delete(b.records, key)
b.byDir, b.mods = nil, nil
}
}
// dirKey names a directory of a repository whichever way the repository is spelled: a build record names it by its clone
// URL, a forge message by "owner/name" (issue 1281).
func dirKey(repository, dir string) string {
return strings.ToLower(delivery.RepositoryName(repository)) + "\x00" + strings.Trim(dir, "/")
}
// Module is the module a build record of the repository names the directory as its path, if one does.
func (b *Builds) Module(repository, dir string) (string, bool) {
b.mu.Lock()
defer b.mu.Unlock()
if b.byDir == nil {
b.byDir = map[string]delivery.BuildRecord{}
for _, r := range b.records {
k := dirKey(r.Repository, r.Path)
if old, ok := b.byDir[k]; !ok || r.At.After(old.At) || (r.At.Equal(old.At) && r.Module < old.Module) {
b.byDir[k] = r
}
}
}
r, ok := b.byDir[dirKey(repository, dir)]
return r.Module, ok
}
// ModuleOf is the module of a changed directory: the one a build record of the repository names it the path of; a
// directory no build record names is the directory's last element (the catalogue's rule for nearly every manifest).
func (b *Builds) ModuleOf(repository, dir string) string {
if m, ok := b.Module(repository, dir); ok {
return m
}
return path.Base(strings.Trim(dir, "/"))
}
// Known is a module the builds know of in a repository: its directory there and its newest successful trunk build
// (delivery.BuildRecord.IsTrunkBuild) if any. The newest key of a module is any state and ref, so it is not what
// finds this (mesh-sdk go/delivery/buildrecord.go NewestKey); the records are.
type Known struct {
Module string
Repository string
Path string
Trunk *delivery.BuildRecord
delete(b.records, key)
}
func newer(a, b delivery.BuildRecord) bool {
return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID)
}
// Modules are the modules built from a repository, by name. A module whose newest successful trunk build is of the
// repository belongs to it with that record's path; a module with no such build belongs to the repository of its newest
// record of any kind.
func (b *Builds) Modules(repository string) []Known {
b.mu.Lock()
defer b.mu.Unlock()
if b.mods == nil {
b.mods = map[string]*moduleBuilds{}
for _, r := range b.records {
m := b.mods[r.Module]
if m == nil {
m = &moduleBuilds{any: r}
b.mods[r.Module] = m
} else if newer(r, m.any) {
m.any = r
}
if r.IsTrunkBuild() && (m.trunk == nil || newer(r, *m.trunk)) {
c := r
m.trunk = &c
}
}
}
var out []Known
for name, m := range b.mods {
k := Known{Module: name, Repository: m.any.Repository, Path: strings.Trim(m.any.Path, "/"), Trunk: m.trunk}
if m.trunk != nil {
k.Repository, k.Path = m.trunk.Repository, strings.Trim(m.trunk.Path, "/")
}
if delivery.SameRepository(k.Repository, repository) {
out = append(out, k)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// TrunkOf is the newest successful trunk build record of a module, in whatever repository it was built from; nil when
// there is none (issue 1727).
func (b *Builds) TrunkOf(module string) *delivery.BuildRecord {
@@ -153,3 +64,27 @@ func (b *Builds) TrunkOf(module string) *delivery.BuildRecord {
}
return best
}
// PathOf is the directory of a module of the repository as its newest build record of that repository names it, its
// newest successful trunk build's first ("." for the repository's root), and false when no record of the repository
// names the module (issue 1614).
func (b *Builds) PathOf(module, repository string) (string, bool) {
var best *delivery.BuildRecord
trunk := false
b.mu.Lock()
defer b.mu.Unlock()
for _, r := range b.records {
if r.Module != module || !delivery.SameRepository(r.Repository, repository) {
continue
}
t := r.IsTrunkBuild()
if best == nil || (t && !trunk) || (t == trunk && newer(r, *best)) {
c := r
best, trunk = &c, t
}
}
if best == nil {
return "", false
}
return pathOf(strings.Trim(best.Path, "/")), true
}
modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go
+5 -4
View File
@@ -3,6 +3,7 @@ package index
import (
"context"
"errors"
"reflect"
"strings"
"testing"
@@ -43,7 +44,7 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t
}
c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want {
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || !reflect.DeepEqual(c.Pulls[0].Fps["avahi"], want) {
t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
}
if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) {
@@ -55,7 +56,7 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t
if err := ix.Apply(context.Background(), again); err != nil {
t.Fatal(err)
}
if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet {
if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.FPSet != c.FPSet {
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet)
}
}
@@ -73,8 +74,8 @@ func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/
func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3
avahi := catalogueManifest(t, "avahi")
// The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory.
other := catalogueManifest(t, "ca-trust")
// The second pull request carries avahi's manifest changed (a line added at its end) for the same directory.
other := avahi + "\n"
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
t.Fatal(err)
modules/mesh-delivery-indexer/internal/index/candidates.go
+16 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"sync"
@@ -46,7 +47,12 @@ type Indexer struct {
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
Manifests Store
Builds *Builds
Logf func(format string, args ...any)
// Named asks the forge for the git object of each named path of the module directories given, at a commit
// (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
// has none.
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
Logf func(format string, args ...any)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
@@ -78,11 +84,15 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
}
}
var digests map[string]string
var held map[string]delivery.Module
if f.Kind == delivery.ForgePull {
var err error
if digests, err = ix.keepManifests(ctx, f); err != nil {
return err
}
if held, err = ix.heldModules(ctx); err != nil {
return err
}
}
if trunkLike[f.Branch] {
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
@@ -101,7 +111,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
}
cur = &c
}
next := ix.next(cur, f, digests)
next := ix.next(cur, f, digests, held)
if next != nil {
ix.withPreviews(ctx, next)
if err := ix.refuseDisagreement(*next); err != nil {
@@ -207,8 +217,8 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
// directory.
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate {
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate {
var pulls []delivery.CandidatePull
var base uint64
if cur != nil {
@@ -225,7 +235,7 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
switch {
case f.DirsKnown:
for _, dir := range f.Dirs {
m := ix.Builds.ModuleOf(f.Repository, dir)
m := moduleOfDir(held, f, dir)
if !slices.Contains(entry.Modules, m) {
entry.Modules = append(entry.Modules, m)
}
@@ -280,7 +290,7 @@ func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error {
first[module] = p
continue
}
if before.Fps[module] != m {
if !reflect.DeepEqual(before.Fps[module], m) {
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
return err
modules/mesh-delivery-indexer/internal/index/index_test.go
+20 -12
View File
@@ -148,27 +148,35 @@ func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the gr
}
}
func TestModuleOfADirectory(t *testing.T) { // 1238/AC3
func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the module of a directory is the indexer's own (I2, ADR 0501 clause 6)
ix, st, _ := newIndexer(t)
// modules/odd is named by a build record of the repository (module odd-name); modules/plain is named by none.
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil {
mods := newMem()
ix.Modules = mods
ctx := context.Background()
// The trunk holds modules/odd as the module odd-name (its manifest names it so); modules/plain is held by none.
merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/odd": "t"})
merged.Manifests["modules/odd"] = `{"module":"odd-name","version":"1"}`
if err := ix.Apply(ctx, merged); err != nil {
t.Fatal(err)
}
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil {
t.Fatal(err)
}
c, _ := read(t, st, "feat/x")
if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" {
t.Errorf("modules = %v, want the record's module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules)
t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules)
}
// A record of another repository does not name this repository's directory.
if m, ok := ix.Builds.Module("novox/other", "modules/a"); ok {
t.Errorf("modules/a of novox/other is %q", m)
held := map[string]delivery.Module{"x": {Repository: "novox/other", Path: "modules/a"}}
if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok {
t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name)
}
// The newest record of a directory wins.
ix.Builds.Put(delivery.BuildKey("renamed", "build-9"), buildRecord(t, "novox/mesh-delivery", "renamed", "modules/a", "2026-10-19T10:00:00Z"))
if m := ix.Builds.ModuleOf("novox/mesh-delivery", "modules/a"); m != "renamed" {
t.Errorf("modules/a = %q, want the newest record's module", m)
// A manifest the pull message carries names its module, whatever directory it is in.
f := pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/odd")
f.Manifests = map[string]string{"modules/odd": `{"module":"renamed","version":"1"}`}
if name := moduleOfDir(held, f, "modules/odd"); name != "renamed" {
t.Errorf("modules/odd = %q, want the module its manifest names", name)
}
}
func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
ix, st, _ := newIndexer(t)
ctx := context.Background()
modules/mesh-delivery-indexer/internal/index/inputs.go
+45 -5
View File
@@ -37,6 +37,8 @@ type moduleInputs struct {
Languages map[string]string
// None2 is why the module has no fingerprint of the new form.
None2 string
// ArtifactPaths are the paths each artifact naming them names, as its manifest names them (delivery.Module.ArtifactPaths).
ArtifactPaths map[string][]string
}
// baseFP answers a base module's new-form fingerprint, or why it has none.
@@ -55,17 +57,55 @@ type baseFP func(module string) (fp, none string)
// here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve
// nothing and have no toolchain. A base is a build.on entry that names a module.
func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs {
m, err := catalogue.ParseManifest([]byte(raw))
return inputsOfNamed(module, raw, tree, locks, nil, base)
}
// inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery
// Forge.Paths, PreviewModule.Paths). A module whose every artifact names its paths takes its tree input from them and its
// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs
// gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it
// none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given.
func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, base baseFP) moduleInputs {
m, err := catalogue.ParseManifestForInputs([]byte(raw))
if err != nil {
return moduleInputs{None2: "the manifest cannot be read: " + err.Error()}
}
if m.Module != module {
return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)}
}
if tree == "" {
return moduleInputs{None2: "the merge names no tree for the module's directory"}
var artifactPaths map[string][]string
if m.Build != nil {
for _, a := range m.Build.Artifacts {
if len(a.Paths) > 0 {
if artifactPaths == nil {
artifactPaths = map[string][]string{}
}
artifactPaths[a.Name] = slices.Clone(a.Paths)
}
}
}
kept := delivery.KeptInputs2{Tree: tree}
var named map[string]string
if paths, every := m.NamedPaths(); every {
var missing []string
named = map[string]string{}
for _, p := range paths {
if objects[p] == "" {
missing = append(missing, p)
continue
}
named[p] = objects[p]
}
if len(missing) > 0 {
return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths}
}
if tree, err = delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil {
return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths}
}
}
if tree == "" {
return moduleInputs{None2: "the merge names no tree for the module's directory", ArtifactPaths: artifactPaths}
}
kept := delivery.KeptInputs2{Tree: tree, Paths: named}
contexts := false
var missing []string
langs := languagesOf(m)
@@ -94,7 +134,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo
}
sort.Strings(kept.Bases)
sort.Strings(missing)
out := moduleInputs{Kept: &kept}
out := moduleInputs{Kept: &kept, ArtifactPaths: artifactPaths}
if len(langs) > 0 {
out.Languages = langs
}
modules/mesh-delivery-indexer/internal/index/live_builds_test.go
+8 -41
View File
@@ -1,7 +1,6 @@
package index
import (
"context"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -51,49 +50,17 @@ func liveIndexer(t *testing.T) (*Indexer, *memStore) {
return ix, mods
}
func TestMergeWritesEveryModuleTheLiveRecordsNameInTheRepository(t *testing.T) { // 1281/AC3
ix, mods := liveIndexer(t)
msg := mergedMsg("novox/mesh-catalog", map[string]string{
"modules/zsh": liveTree, "modules/mesh-delivery": "newtree", "modules/mailu": "newtree", "modules/sonarr": "newtree",
})
if err := ix.Apply(context.Background(), msg); err != nil {
t.Fatal(err)
}
for _, name := range []string{"zsh", "mesh-delivery", "mailu"} {
m, ok := moduleOf(t, mods, name)
if !ok {
t.Errorf("%s: no module key written", name)
continue
}
if (m.FP == "") == (m.None == "") || m.Repository != "novox/mesh-catalog" || m.Commit != merge {
t.Errorf("%s: module = %+v, want an fp or a none, repository novox/mesh-catalog, commit %s", name, m, merge)
}
}
// the tree of zsh at the merge is the tree its inputs record, so its fingerprint is the one those inputs give
var in delivery.SourceInputs
in.Module, in.Tree = "zsh", liveTree
in.Bases = []string{"sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"}
in.Toolchains = map[string]string{"code": "typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"}
want, _ := delivery.SourceFingerprint(in)
if m, _ := moduleOf(t, mods, "zsh"); m.FP != want {
t.Errorf("zsh fp = %q, want %q", m.FP, want)
}
if _, ok := mods.vals["sonarr"]; ok {
t.Error("the key of a module of mesh-media-catalog was written at a merge of mesh-catalog")
}
}
func TestChangedDirectoryMapsToTheModuleItsLiveRecordNamesByPath(t *testing.T) { // 1281/AC4
func TestTheLiveRecordsGiveTheDirectoryOfAModuleOfTheirRepository(t *testing.T) { // 1614 (I3: the records name directories, never list modules)
ix, _ := liveIndexer(t)
for dir, want := range map[string]string{"modules/zsh": "zsh", "modules/mesh-delivery": "mesh-delivery", "modules/mailu/": "mailu"} {
if got, ok := ix.Builds.Module("novox/mesh-catalog", dir); !ok || got != want {
t.Errorf("Module(novox/mesh-catalog, %q) = %q, %v; want %q", dir, got, ok, want)
for module, want := range map[string]string{"zsh": "modules/zsh", "mesh-delivery": "modules/mesh-delivery", "mailu": "modules/mailu"} {
if got, ok := ix.Builds.PathOf(module, "novox/mesh-catalog"); !ok || got != want {
t.Errorf("PathOf(%s, novox/mesh-catalog) = %q, %v; want %q", module, got, ok, want)
}
}
if got, ok := ix.Builds.Module("novox/mesh-catalog", "modules/sonarr"); ok {
t.Errorf("Module found %q for a directory of mesh-media-catalog under mesh-catalog", got)
if got, ok := ix.Builds.PathOf("sonarr", "novox/mesh-catalog"); ok {
t.Errorf("PathOf found %q for a module of mesh-media-catalog under mesh-catalog", got)
}
if got := ix.Builds.ModuleOf("novox/mesh-media-catalog", "modules/sonarr"); got != "sonarr" {
t.Errorf("ModuleOf(mesh-media-catalog, modules/sonarr) = %q, want sonarr", got)
if got, ok := ix.Builds.PathOf("sonarr", "novox/mesh-media-catalog"); !ok || got != "modules/sonarr" {
t.Errorf("PathOf(sonarr, mesh-media-catalog) = %q, %v; want modules/sonarr", got, ok)
}
}
modules/mesh-delivery-indexer/internal/index/manifests.go
+1 -16
View File
@@ -2,27 +2,12 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests
// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the
// module a build record names the directory the path of.
func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string {
var m struct {
Module string `json:"module"`
}
if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" {
return m.Module
}
name, _ := ix.Builds.Module(repository, dir)
return name
}
// keepManifest writes manifests.<digest> for a manifest as the forge answered it, create-once (issue 1449): a key that
// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for
// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the
@@ -66,7 +51,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string
func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
digests := make(map[string]string, len(f.Manifests))
for dir, raw := range f.Manifests {
module := ix.moduleNameOf(f.Repository, dir, raw)
module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module
digest, err := ix.keepManifest(ctx, module, raw)
switch {
case err == nil:
modules/mesh-delivery-indexer/internal/index/manifests_test.go
+3 -2
View File
@@ -96,7 +96,8 @@ func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a
if len(manifests.vals) != 0 || len(logs) == 0 {
t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs)
}
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" {
t.Errorf("module = %+v (%v), want no manifest digest", mod, ok)
// A manifest that is not JSON names no module (I1): nothing is written for its directory.
if mod, ok := moduleOf(t, mods, "avahi"); ok {
t.Errorf("module = %+v, want none written for a manifest naming no module", mod)
}
}
modules/mesh-delivery-indexer/internal/index/modules.go
+91 -60
View File
@@ -2,79 +2,71 @@ package index
import (
"context"
"sort"
"strings"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The reasons a module has no fingerprint (ADR 0437 §5; issue 1209's decision).
const (
// NoneNarrow is for a module whose image compiles a Go program from a context: the context is the hash of the build
// source the builder handed it, not a git tree.
NoneNarrow = "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4"
// NoneContext is for a context that is a repository's root tree.
NoneContext = "an artifact reads the root tree of a repository the indexer does not know the tree of before release 4"
// NoneNoBuild is for a module no successful trunk build recorded the inputs of.
NoneNoBuild = "no successful trunk build recorded its inputs"
)
// fingerprintOf is the module's fingerprint at the merge, or none and why: its directory's tree from the merged
// message and the bases, contexts and toolchains its newest successful trunk build recorded.
func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
if k.Trunk == nil {
return "", NoneNoBuild
}
in := *k.Trunk.SourceInputs
artifacts := make([]string, 0, len(in.Contexts))
for a := range in.Contexts {
artifacts = append(artifacts, a)
}
sort.Strings(artifacts)
for _, a := range artifacts {
if strings.HasPrefix(in.Contexts[a], "narrow:") {
return "", NoneNarrow
}
}
if len(artifacts) > 0 {
return "", NoneContext
}
in.Tree = trees[k.Path] // the directory "" is the repository's root
return delivery.SourceFingerprint(in)
}
// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and
// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it
// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already
// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it
// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449,
// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build
// module reports its toolchain.
// applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that
// holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest,
// its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as
// the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is
// computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain.
//
// The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue
// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds is not written again.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
return err
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
objects, namedErr := ix.namedObjects(ctx, f)
overlay := map[string]delivery.Module{}
for _, k := range ix.Builds.Modules(f.Repository) {
key, err := delivery.ModulesKey(k.Module)
if err != nil {
ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, k.Module, err)
dirOf := map[string]string{}
for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) {
raw := f.Manifests[dir]
name := manifestModule(raw)
if name == "" {
ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir)
continue
}
fp, none := fingerprintOf(k, f.Trees)
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path], Path: pathOf(k.Path)}
// The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which
// names what is missing, from its manifest and lock hashes (inputs.go).
raw, has := f.Manifests[k.Path]
switch {
case !has:
next.None2 = None2NoManifest
case next.Manifest == "":
key, err := delivery.ModulesKey(name)
if err != nil {
ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err)
continue
}
cur, has := held[key]
if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository)
continue
}
if other, dup := dirOf[name]; dup {
if !has || cur.Path != pathOf(dir) {
ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other)
continue
}
ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir)
}
dirOf[name] = dir
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)}
if next.Manifest == "" {
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
default:
in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx))
next.Inputs2, next.None2 = in.Kept, in.None2
} else {
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx))
next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept == nil && namedErr[dir] != "" {
next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir]
}
}
overlay[key] = next
}
@@ -118,3 +110,42 @@ func pathOf(dir string) string {
}
func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) {
objects := map[string]map[string]string{}
need := map[string][]string{}
for dir, raw := range f.Manifests {
m, err := catalogue.ParseManifestForInputs([]byte(raw))
if err != nil {
continue // inputsOfNamed says why
}
paths, every := m.NamedPaths()
if !every {
continue
}
if carried := f.Paths[dir]; len(carried) > 0 {
objects[dir] = carried
continue
}
need[dir] = paths
}
if len(need) == 0 || ix.Named == nil {
return objects, nil
}
got, err := ix.Named(ctx, f.Repository, f.Merge, need)
if err != nil {
words := map[string]string{}
for dir := range need {
words[dir] = err.Error()
}
ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err)
return objects, words
}
for dir := range need {
objects[dir] = got[dir]
}
return objects, nil
}
modules/mesh-delivery-indexer/internal/index/modules_test.go
+88 -136
View File
@@ -3,6 +3,7 @@ package index
import (
"context"
"encoding/json"
"path"
"strings"
"testing"
@@ -51,8 +52,22 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2
return [2]any{k, v}
}
// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its
// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that
// holds a module.json).
func mergedMsg(repo string, trees map[string]string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees}
manifests := map[string]string{}
for dir := range trees {
manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}`
}
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
}
func nameOfDir(dir string) string {
if dir == "" {
return "r"
}
return path.Base(dir)
}
func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
@@ -70,126 +85,6 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
func TestModuleFingerprintFromTheMergeTreeAndTheRecordedInputs(t *testing.T) { // 1239/AC1
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "modules/b": "other"})); err != nil {
t.Fatal(err)
}
want, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
if none != "" {
t.Fatal(none)
}
m, ok := moduleOf(t, mods, "a")
if !ok || m.FP != want || m.None != "" || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Src != 0 {
t.Errorf("module = %+v (%v), want fp %s", m, ok, want)
}
if _, ok := mods.vals["b"]; ok {
t.Error("a key was written for a directory no build names")
}
}
func TestMergeOfTheRecordedTreeReproducesTheRecordedFingerprint(t *testing.T) { // 1239/AC2
// mailu, build-1792391134620131370: mesh-catalog at b4527d9091e972f28cb365623d9280045e7741a3, dir modules/mailu. Its
// inputs and recorded fingerprint are the live ones mesh-sdk go/delivery/fingerprint_test.go TestFingerprintLiveBuilds
// captured (its tree is `git rev-parse <commit>:modules/mailu`, its bases from the build's [bases] log line).
const tree = "cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e"
const recorded = "src1:539f2e64000d436c572ef53b40e524012455945f8c5a1c6bb173ee0707e88ef4"
inputs := `{"Module":"mailu","Tree":"` + tree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` +
`"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}`
ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "", "2026-10-19T09:00:00Z", inputs))
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-catalog", map[string]string{"modules/mailu": tree})); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "mailu"); !ok || m.FP != recorded {
t.Errorf("module = %+v (%v), want fp %s", m, ok, recorded)
}
}
func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1
// A Go program's build source as the builder records it (build-agent's record, mesh-sdk go/delivery/testdata):
// the context "server" is "narrow:" and a hash.
inputs := `{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""}`
ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", inputs))
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-controller", map[string]string{"modules/build-agent": "newtree"})); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "build-agent")
if !ok || m.FP != "" || !strings.Contains(m.None, "Go program") || !strings.Contains(m.None, "release 4") {
t.Errorf("module = %+v (%v), want none naming the Go program's build source and release 4", m, ok)
}
}
func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4
inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":"the build resolves packages when built"}`
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs))
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "the build resolves packages when built" {
t.Errorf("module = %+v (%v)", m, ok)
}
}
func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5
inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":{"web":"f00dcafef00dcafef00dcafef00dcafef00dcafe"},"Toolchains":null,"Unpinned":""}`
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs))
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
m, ok := moduleOf(t, mods, "a")
if !ok || m.FP != "" || !strings.Contains(m.None, "root tree") || !strings.Contains(m.None, "release 4") {
t.Errorf("module = %+v (%v)", m, ok)
}
}
func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/other": "t"}))
if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "no tree" {
t.Errorf("module = %+v (%v)", m, ok)
}
}
func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7
ix, mods := modulesIndexer(t,
rec(t, "build-1", "failed-only", "novox/mesh-delivery", "modules/f", "main", "failed", "2026-10-19T09:00:00Z", aInputs),
rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "", "2026-10-19T09:00:00Z", aInputs),
rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "", "2026-10-19T09:00:00Z", ""),
)
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/f": "t", "modules/g": "t", "modules/h": "t"}))
for _, name := range []string{"failed-only", "branch-only", "no-inputs"} {
if m, ok := moduleOf(t, mods, name); !ok || m.FP != "" || !strings.Contains(m.None, "no successful trunk build") {
t.Errorf("%s: module = %+v (%v)", name, m, ok)
}
}
}
func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest key is only a hint)
older := strings.Replace(aInputs, `"Bases":["registry.example/base@sha256:aaa"]`, `"Bases":["registry.example/base@sha256:old"]`, 1)
ix, mods := modulesIndexer(t,
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", older),
rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T10:00:00Z", aInputs),
rec(t, "build-3", "a", "novox/mesh-delivery", "modules/a", "main", "failed", "2026-10-19T11:00:00Z", older),
rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "", "2026-10-19T12:00:00Z", older),
)
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
if m, _ := moduleOf(t, mods, "a"); m.FP != want {
t.Errorf("fp = %s, want the 10:00 trunk build's %s", m.FP, want)
}
}
func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8
ix, mods := modulesIndexer(t,
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1)))
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/c": "t"}))
if _, ok := mods.vals["c"]; ok {
t.Error("the key of a module of another repository was written")
}
if _, ok := mods.vals["a"]; !ok {
t.Error("the key of the merged repository's module was not written")
}
}
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
@@ -211,21 +106,6 @@ func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5
}
}
func TestARecordAsTheControllerWritesItGivesAFingerprint(t *testing.T) { // 1281/AC8
// The layout of mesh-controller internal/chain/records.go Build (commit 1f708e2d8d72): an outcome has no state key, a build
// asked by a plan after a merge no ref, and the repository is the clone URL.
raw := `{"id":"build-1","repository":"http://novox.internal:20000/novox/mesh-delivery.git","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20",` +
`"path":"modules/a","source-fingerprint":"src1:ignored","source-inputs":` + aInputs + `,"at":"2026-10-19T09:00:00Z"}`
ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("a", "build-1"), []byte(raw)})
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "a")
if !ok || m.FP == "" || m.None != "" {
t.Errorf("module = %+v (%v), want a fingerprint, not none", m, ok)
}
}
func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2
ix, mods := modulesIndexer(t,
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
@@ -263,3 +143,75 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means
t.Errorf("module = %+v (%v), want path %q", m, ok, delivery.RootPath)
}
}
func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6
ix, mods, _ := manifestsIndexer(t) // no build record of any module
avahi := catalogueManifest(t, "avahi")
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "avahi")
if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" {
t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok)
}
if m.FP != "" || m.None != "" {
t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m)
}
if m.FP2 == "" && m.None2 == "" {
t.Errorf("module = %+v, want a new-form fingerprint or its reason", m)
}
}
func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1
ix, mods := modulesIndexer(t, rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/b": "t"})
f.Trees["modules/c"] = "t" // a directory whose manifest the message does not carry
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
for name, want := range map[string]bool{"a": true, "b": true, "c": false} {
if _, ok := mods.vals[name]; ok != want {
t.Errorf("modules.%s written %v, want %v", name, ok, want)
}
}
}
func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7)
ix, mods := modulesIndexer(t)
var logs []string
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
ctx := context.Background()
if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil {
t.Fatal(err)
}
before := string(mods.vals["a"])
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil {
t.Fatal(err)
}
if string(mods.vals["a"]) != before || len(logs) == 0 {
t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs)
}
}
func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { // I1
ix, mods := modulesIndexer(t)
var logs []string
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "other/a": "t2"})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "a"); !ok || m.Path != "modules/a" || len(logs) == 0 {
t.Errorf("module = %+v (%v), logs %v: want the first directory by name, and the other said", m, ok, logs)
}
// Once held, the directory the bucket holds stays the module's.
f = mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "aaa/a": "t3"})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" {
t.Errorf("module = %+v, want the held directory kept", m)
}
}
modules/mesh-delivery-indexer/internal/index/named_test.go
+133
View File
@@ -0,0 +1,133 @@
package index
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0499 clauses 2 and 3 (I7): a module whose every artifact names its paths takes its tree input from them, by mesh-sdk's
// one function, and keeps their objects, so a computation from kept inputs gives the merge's value. The manifest is
// mesh-sdk's module.json at 9cb4d59 (its lib names src, test, package.json, package-lock.json and tsconfig.json).
var sdkObjects = map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
func TestAModuleNamingItsPathsTakesItsTreeFromThemAndKeepsTheirObjects(t *testing.T) { // I7, ADR 0499 clause 2
ix, mods, _ := manifestsIndexer(t)
raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, map[string]map[string]string{"": {"package-lock.json": avahiGoSum}})
f.Paths = map[string]map[string]string{"": sdkObjects}
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "mesh-sdk")
if !ok || m.Inputs2 == nil {
t.Fatalf("module = %+v (%v)", m, ok)
}
want, err := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects)
if err != nil {
t.Fatal(err)
}
if m.Inputs2.Tree != want || !reflect.DeepEqual(m.Inputs2.Paths, sdkObjects) {
t.Errorf("inputs = %+v, want the named tree %s and the objects kept", m.Inputs2, want)
}
if got := m.ArtifactPaths["lib"]; strings.Join(got, ",") != "src,test,package.json,package-lock.json,tsconfig.json" {
t.Errorf("artifact paths = %v, want lib's paths as its manifest names them", m.ArtifactPaths)
}
// Recomputed from what is kept, with no merge, the module is the same (ADR 0499 clause 2).
before := string(mods.vals["mesh-sdk"])
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if string(mods.vals["mesh-sdk"]) != before {
t.Errorf("a recompute changed the module: %s, was %s", mods.vals["mesh-sdk"], before)
}
}
func TestAMessageWithoutTheNamedPathsObjectsGivesTheModuleNoneNamingThem(t *testing.T) { // I7, ADR 0499 clause 3
ix, mods, _ := manifestsIndexer(t)
raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "mesh-sdk")
if m.FP2 != "" || !strings.Contains(m.None2, "named paths") || !strings.Contains(m.None2, "tsconfig.json") {
t.Errorf("module = %+v, want none naming the paths the message did not carry", m)
}
}
func TestAModuleNamingNoPathsKeepsItsDirectorysTree(t *testing.T) { // I7, ADR 0499 clause 2: every other module is unchanged
in := inputsOfNamed("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, map[string]string{"x": "1111111111111111111111111111111111111111"}, noBases)
if in.Kept == nil || in.Kept.Tree != "avahitree" || in.Kept.Paths != nil {
t.Errorf("inputs = %+v, want the directory's tree and no paths", in.Kept)
}
}
func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) { // I8, ADR 0499 clause 2 (the merge preview)
ix, _, _ := manifestsIndexer(t)
raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
digest, err := ix.keepManifest(context.Background(), "mesh-sdk", raw)
if err != nil {
t.Fatal(err)
}
named, _ := delivery.NamedTree(digest, sdkObjects)
with := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}}
without := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: "roottree", Manifest: digest}}}
if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without); !strings.Contains(got.None2, "named paths") {
t.Errorf("without the objects: %+v, want none naming the named paths", got)
}
got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with)
if strings.Contains(got.None2, "named paths") || got.Manifest != digest {
t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got)
}
if got.FP2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) {
t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got)
}
}
func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths
ix, mods, _ := manifestsIndexer(t)
type ask struct {
repository, sha string
paths map[string][]string
}
var asked []ask
ix.Named = func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
asked = append(asked, ask{repository, sha, paths})
return map[string]map[string]string{"": sdkObjects}, nil
}
raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree", "modules/a": "t"}), map[string]string{"": raw, "modules/a": `{"module":"a","version":"1"}`}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if len(asked) != 1 || asked[0].repository != "novox/mesh-sdk" || asked[0].sha != merge || len(asked[0].paths) != 1 ||
strings.Join(asked[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" {
t.Fatalf("asked %+v, want the root's named paths of the merge commit, once, and nothing for a module naming none", asked)
}
m, _ := moduleOf(t, mods, "mesh-sdk")
want, _ := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects)
if m.Inputs2 == nil || m.Inputs2.Tree != want {
t.Errorf("module = %+v, want the named tree %s", m, want)
}
}
func TestAForgeThatCannotSayTheObjectsGivesTheModuleNoneWithItsWords(t *testing.T) { // I6
ix, mods, _ := manifestsIndexer(t)
ix.Named = func(context.Context, string, string, map[string][]string) (map[string]map[string]string, error) {
return nil, errors.New("gitea: the path tsconfig.json of \"\" is not at the commit")
}
raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "mesh-sdk"); m.FP2 != "" || !strings.Contains(m.None2, "is not at the commit") {
t.Errorf("module = %+v, want none with the forge's words", m)
}
}
modules/mesh-delivery-indexer/internal/index/newform_test.go
+8 -23
View File
@@ -7,17 +7,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go).
func wantAvahiFP(t *testing.T) string {
t.Helper()
fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
if none != "" {
t.Fatal(none)
}
return fp
}
func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1
avahi := catalogueManifest(t, "avahi")
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z",
@@ -31,16 +20,14 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t
if !ok {
t.Fatal("no module avahi")
}
todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
t.Errorf("module = %+v, want its manifest digest and inputs", m)
}
if m.None2 != "toolchain not yet reported" || m.FP2 != "" {
t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2)
}
if m.FP != todays || m.None != "" {
t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays)
if m.FP != "" || m.None != "" {
t.Errorf("fp = %q none = %q, want no value of today's form (I4)", m.FP, m.None)
}
}
@@ -55,17 +42,15 @@ func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T)
}
}
func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3
func TestAMergedMessageWithoutManifestsWritesNoModule(t *testing.T) { // 1444/AC3 as R4e has it: the modules are the manifests' (I1)
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
f.Manifests = nil
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "a")
if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" {
t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok)
}
if len(manifests.vals) != 0 {
t.Errorf("manifests holds %d keys", len(manifests.vals))
if len(mods.vals) != 0 || len(manifests.vals) != 0 {
t.Errorf("modules %d keys, manifests %d keys, want none: a message naming no manifest names no module", len(mods.vals), len(manifests.vals))
}
}
modules/mesh-delivery-indexer/internal/index/own.go
+81
View File
@@ -0,0 +1,81 @@
package index
import (
"context"
"encoding/json"
"fmt"
"path"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// From R4e the indexer lists a repository's modules from what it holds itself (novox/hq ADR 0501 clause 6, issue 1851):
// a merged message's manifests name the modules of the merge, and the modules bucket names the modules it wrote before,
// with their repository and directory. The controller's build records name none of them (ADR 0453 clause 5 keeps only
// the builder's own input from them until R4g, builds.go).
// manifestModule is the module a manifest names itself (its `module`), or "" when it names none or is not JSON. Only the
// name is read here; the manifest is read whole by the catalogue's parser (inputs.go).
func manifestModule(raw string) string {
var m struct {
Module string `json:"module"`
}
if json.Unmarshal([]byte(raw), &m) != nil {
return ""
}
return m.Module
}
// heldModules are the values the modules bucket holds, by key (the module's name). A value that is not a module is left
// out and said.
func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module, error) {
out := map[string]delivery.Module{}
if ix.Modules == nil {
return out, nil
}
keys, err := ix.Modules.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("listing the modules: %w", err)
}
for _, key := range keys {
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", key, err)
}
if !found {
continue
}
m, err := delivery.DecodeModule(value)
if err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
}
out[key] = m
}
return out, nil
}
// moduleAt is the module whose directory in the repository is dir, as the modules bucket holds it.
func moduleAt(held map[string]delivery.Module, repository, dir string) (string, bool) {
want := pathOf(strings.Trim(dir, "/"))
for name, m := range held {
if m.Path == want && delivery.SameRepository(m.Repository, repository) {
return name, true
}
}
return "", false
}
// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else
// the one the modules bucket holds at that directory of the repository, else the directory's last element (the
// catalogue's rule for nearly every manifest).
func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string {
if name := manifestModule(f.Manifests[dir]); name != "" {
return name
}
if name, ok := moduleAt(held, f.Repository, dir); ok {
return name
}
return path.Base(strings.Trim(dir, "/"))
}
modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go
+67
View File
@@ -0,0 +1,67 @@
package index
import (
"context"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Issue 1614: a modules key written before the indexer learned paths (mesh-tools and node-tools live) gets its directory
// from the module's newest build record without waiting for a merge of its repository, once, changing no fingerprint. The
// read of the controller's build records for it goes in R4g with build-agent (the lead's decision, 2026-10-22).
func heldValue(t *testing.T, st *memStore, name, path string) {
t.Helper()
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-tools", Commit: "e0fadc43d9dda0bc24bd19bb05b021dd87da8582",
FP2: "src2:" + hex64('2'), Manifest: hex64('9'), Path: path})
if err != nil {
t.Fatal(err)
}
st.put(name, data)
}
func hex64(c byte) string {
b := make([]byte, 64)
for i := range b {
b[i] = c
}
return string(b)
}
func TestAnEmptyPathIsFilledOnceFromTheModulesNewestBuildRecord(t *testing.T) { // 1614
ix, mods := modulesIndexer(t,
[2]any{delivery.BuildKey("mesh-tools", "build-1"), buildRecord(t, "novox/mesh-tools", "mesh-tools", "", "2026-10-19T09:00:00Z")},
[2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "node-tools", "2026-10-19T09:00:00Z")})
heldValue(t, mods, "mesh-tools", "")
heldValue(t, mods, "node-tools", "")
before, _ := moduleOf(t, mods, "node-tools")
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
for name, want := range map[string]string{"mesh-tools": delivery.RootPath, "node-tools": "node-tools"} {
m, _ := moduleOf(t, mods, name)
if m.Path != want || m.FP2 != before.FP2 || m.Commit != before.Commit {
t.Errorf("%s = %+v, want path %q and nothing else changed", name, m, want)
}
}
writes := mods.writes
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if mods.writes != writes {
t.Errorf("the paths were written again (%d writes, was %d): once only", mods.writes, writes)
}
}
func TestASetPathIsNeverOverwrittenByABuildRecord(t *testing.T) { // 1614
ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "elsewhere", "2026-10-19T09:00:00Z")})
heldValue(t, mods, "node-tools", "node-tools")
writes := mods.writes
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "node-tools"); m.Path != "node-tools" || mods.writes != writes {
t.Errorf("module = %+v after %d writes (was %d), want its path kept and nothing written", m, mods.writes, writes)
}
}
modules/mesh-delivery-indexer/internal/index/predefined_test.go
+29
View File
@@ -0,0 +1,29 @@
package index
import (
"strings"
"testing"
)
// Issue 1751: a manifest whose config value falls back to a predefined key (novox/hq ADR 0348) is read by the indexer, so
// the module gets its new-form inputs. The manifests are claude-code's, dunst's and xterm's module.json from mesh-catalog
// at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) { // 1751
for _, module := range []string{"claude-code", "dunst", "xterm"} {
raw := fixture(t, "catalogue-835bf22b-"+module)
if !strings.Contains(raw, `"predefined"`) {
t.Fatalf("%s: the fixture names no predefined key", module)
}
in := inputsOf(module, raw, "newtree", nil, func(string) (string, string) { return "src2:" + hex64('1'), "" })
if in.Kept == nil || strings.Contains(in.None2, "cannot be read") {
t.Errorf("%s: inputs = %+v, want the manifest read and its inputs kept", module, in)
}
}
}
func TestAPredefinedKeyOfAnotherTypeThanItsValueIsStillRefused(t *testing.T) { // 1751: the read judges what it can
raw := strings.Replace(fixture(t, "catalogue-835bf22b-xterm"), `"predefined": "font-size"`, `"predefined": "font-size", "trusted": "yes"`, 1)
if in := inputsOf("xterm", raw, "newtree", nil, func(string) (string, string) { return "", "none" }); in.Kept != nil {
t.Errorf("inputs = %+v, want a manifest the catalogue refuses still refused", in)
}
}
modules/mesh-delivery-indexer/internal/index/previews.go
+31 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"maps"
"reflect"
"slices"
"strings"
@@ -26,7 +27,7 @@ const (
NonePreviewNoModule = "the merge preview names no manifest for the module's directory"
// NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold.
NonePreviewNoManifest = "the manifest of the merge is not kept"
// NoneNoDirectory is for a module whose directory in its repository the builds do not tell.
// NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells.
NoneNoDirectory = "the module's directory in its repository is not known"
)
@@ -143,9 +144,27 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
if !found && !stale {
continue
}
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
dirs := map[string]string{}
for _, k := range ix.Builds.Modules(p.Repository) {
dirs[k.Module] = k.Path
if held, err := ix.heldModules(ctx); err == nil {
for name, m := range held {
if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" {
dirs[name] = m.Path
}
}
} else {
ix.logf("the modules cannot be listed, so no touched module's directory is known: %v", err)
}
for _, m := range p.Modules {
if _, known := dirs[m]; known || p.Fps[m].Manifest == "" {
continue
}
for dir, pm := range pv.Modules {
if pm.Manifest == p.Fps[m].Manifest {
dirs[m] = dir
}
}
}
fps := maps.Clone(p.Fps)
for _, m := range p.Modules {
@@ -166,12 +185,12 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
if fps == nil {
fps = map[string]delivery.CandidateModule{}
}
if fps[m] != next {
if !reflect.DeepEqual(fps[m], next) {
fps[m] = next
changed = true
}
}
if changed || !maps.Equal(fps, p.Fps) {
if changed || !maps.EqualFunc(fps, p.Fps, func(a, b delivery.CandidateModule) bool { return reflect.DeepEqual(a, b) }) {
p.Fps = fps
}
}
@@ -189,6 +208,9 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
if !ok && dir == "" {
pm, ok = pv.Modules[delivery.RootPath]
}
if !ok && dir == delivery.RootPath {
pm, ok = pv.Modules[""]
}
if !ok {
return delivery.CandidateModule{None2: NonePreviewNoModule}
}
@@ -207,7 +229,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
out.None2 = NonePreviewNoManifest
return out
}
in := inputsOf(m, kept.JSON, pm.Tree, pm.Locks, ix.baseFP(ctx))
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, ix.baseFP(ctx))
if in.Kept == nil {
out.None2 = in.None2
return out
@@ -248,5 +270,8 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
return out
}
out.FP2 = fp
if len(inputs.Paths) > 0 {
out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4)
}
return out
}
modules/mesh-delivery-indexer/internal/index/source.go
+31 -25
View File
@@ -34,7 +34,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing.
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
ix.mu.Lock()
if rev < ix.source.rev {
@@ -46,34 +47,39 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
if ix.Modules == nil {
return ix.RecomputeCandidates(ctx)
}
repositories := slices.Sorted(maps.Keys(s.Heads))
for _, repository := range repositories {
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
for _, key := range slices.Sorted(maps.Keys(held)) {
repository := ""
for r := range s.Heads {
if delivery.SameRepository(r, held[key].Repository) {
repository = r
}
}
if repository == "" {
continue
}
head := s.Heads[repository]
for _, k := range ix.Builds.Modules(repository) {
key, err := delivery.ModulesKey(k.Module)
if err != nil {
ix.logf("%s: the module %q has no usable key, skipped: %v", repository, k.Module, err)
continue
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
if err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
return nil, false, nil
}
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
return nil, false, nil
}
cur.Src = rev
data, err := delivery.EncodeModule(cur)
return data, false, err
})
cur, err := delivery.DecodeModule(value)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
return nil, false, nil
}
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
return nil, false, nil
}
cur.Src = rev
data, err := delivery.EncodeModule(cur)
return data, false, err
})
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
}
// A trunk that moved makes the previews made onto the old one out of date.
modules/mesh-delivery-indexer/internal/index/source_test.go
+4 -5
View File
@@ -15,8 +15,7 @@ func sourceOf(repo, head string) delivery.Source {
}
func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs))
ix, mods := modulesIndexer(t)
ctx := context.Background()
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
t.Fatal(err)
@@ -63,7 +62,7 @@ func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T)
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.FP == "" {
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.Path != "modules/a" {
t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok)
}
}
@@ -75,8 +74,8 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.FP == "" {
t.Fatalf("module a = %+v (%v), want a fingerprint with Src 0", m, ok)
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.Path != "modules/a" {
t.Fatalf("module a = %+v (%v), want the merge written with Src 0", m, ok)
}
// A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0.
if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil {
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json
Vendored
+249
View File
@@ -0,0 +1,249 @@
{
"module": "claude-code",
"version": "1",
"moved": true,
"slug": "agent",
"capabilities": [
"package-manager"
],
"requires": [
"mcp-endpoint",
{
"name": "gitea-read",
"optional": true
}
],
"binds": {
"mcp-endpoint": "${dir:state}/mcp-endpoint.json"
},
"secrets": {
"gitea-read": "${dir:state}/gitea-read.secret"
},
"uses": [
"operator-channel"
],
"state": [
"servers",
"holdings",
"config",
"proposals",
"questions",
"memory",
"waits"
],
"reads": [
"claude-licence-manager.bindings"
],
"tools": [
"claude_code_status",
"claude_code_render",
"claude_code_guard",
"claude_code_pull",
"claude_code_grant",
"claude_code_add_api_key",
"claude_code_registrations",
"claude_code_memory",
"claude_code_mcp_list",
"claude_code_mcp_register",
"claude_code_mcp_unregister",
"claude_code_skill_list",
"claude_code_skill_register",
"claude_code_skill_unregister",
"claude_code_agent_list",
"claude_code_agent_register",
"claude_code_agent_unregister",
"claude_code_command_list",
"claude_code_command_register",
"claude_code_command_unregister",
"claude_code_hook_list",
"claude_code_hook_register",
"claude_code_hook_unregister",
"claude_code_output_style_list",
"claude_code_output_style_register",
"claude_code_output_style_unregister",
"claude_code_instructions_list",
"claude_code_instructions_register",
"claude_code_instructions_unregister",
"claude_code_instructions_propose",
"claude_code_proposals",
"claude_code_ask",
"claude_code_answer",
"claude_code_wait",
"claude_code_settings_get",
"claude_code_settings_set",
"claude_code_settings_clear",
"claude_code_permission_add",
"claude_code_permission_remove",
"claude_code_config_list",
"claude_code_config_show",
"claude_code_config_status",
"claude_code_config_import",
"claude_code_home_show",
"claude_code_home_remove",
"claude_code_home_removed",
"claude_code_home_restore",
"claude_code_judge",
"claude_code_judged"
],
"data": {
"own": [
{
"id": "agent-home",
"path": "${dir:agent-home}",
"class": "valuable",
"why": "the operator's agent's own files: its memory, history and projects"
}
]
},
"config": {
"role": {
"type": "string",
"description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.",
"trusted": true,
"default": ""
},
"mcp_servers": {
"type": "object",
"description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.",
"trusted": true,
"default": {}
},
"managed_settings": {
"type": "object",
"description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.",
"trusted": true,
"predefined": "agent-managed-settings",
"default": {}
},
"instructions": {
"type": "object",
"description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
},
"output_styles": {
"type": "object",
"description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
},
"skills": {
"type": "object",
"description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.",
"trusted": true,
"default": {}
},
"commands": {
"type": "object",
"description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
},
"agents": {
"type": "object",
"description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
"trusted": true,
"default": {}
}
},
"resources": [
{
"id": "package",
"type": "package",
"package": "claude-code"
},
{
"id": "managed",
"type": "directory",
"path": "/etc/claude-code",
"mode": "0755",
"unknown": "program"
},
{
"id": "start",
"type": "file",
"path": "/usr/local/bin/claude-agent",
"mode": "0755",
"template": "files/claude-agent"
},
{
"id": "agent-account-name",
"type": "file",
"path": "/etc/claude-code/agent-account",
"mode": "0644",
"template": "files/agent-account"
},
{
"id": "agent-account",
"type": "user",
"name": "${machine:agent-account}",
"home": "${machine:agent-home}",
"root": "${machine:agent-root}"
},
{
"id": "agent-home",
"type": "directory",
"path": "${machine:agent-home}/.claude",
"mode": "0700",
"owner": "${machine:agent-account}",
"unknown": "module"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "${machine:account}",
"place": ".",
"unknown": "program"
},
{
"id": "facts",
"type": "file",
"path": "${dir:state}/facts.json",
"mode": "0600",
"owner": "${machine:account}",
"template": "files/facts.json"
},
{
"id": "settings",
"type": "file",
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "${machine:account}",
"takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"],
"merge": "json",
"template": "files/settings.json"
}
],
"items": [
{
"collection": "shell:zsh:normal",
"template": "files/zsh-normal"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/claude-code",
"binary": "claude-code",
"loads": [
"claude-code"
],
"env": {
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
}
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json
Vendored
+123
View File
@@ -0,0 +1,123 @@
{
"module": "dunst",
"version": "1",
"moved": true,
"capabilities": [
"package-manager"
],
"claims": [
{
"name": "node-notifier",
"scope": "node",
"serves": [
"send",
"history"
]
}
],
"tools": [
"dunst_pause",
"dunst_resume",
"dunst_close_all",
"dunst_rules",
"dunst_count",
"dunst_reload"
],
"config": {
"font-size": {
"type": "number",
"description": "The size of a notification's text, in points of Inter: 10 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
"minimum": 6,
"maximum": 72,
"predefined": "font-size",
"default": 10
},
"width": {
"type": "number",
"description": "The width of a notification, in pixels: 250 fits a title of about forty characters at 10 points, and a larger font needs a wider notification.",
"minimum": 50,
"maximum": 4000,
"default": 250
}
},
"resources": [
{
"id": "package",
"type": "package",
"package": "dunst"
},
{
"id": "client",
"type": "package",
"package": "libnotify"
},
{
"id": "configuration-dir",
"type": "directory",
"path": "${machine:account-home}/.config/dunst",
"owner": "${machine:account}",
"mode": "0755",
"unknown": "move-out"
},
{
"id": "dropins",
"type": "directory",
"path": "${machine:account-home}/.config/dunst/dunstrc.d",
"owner": "${machine:account}",
"mode": "0755",
"unknown": "move-out",
"knows": ["50-slack.conf"]
},
{
"id": "configuration",
"trusted": false,
"type": "file",
"path": "${machine:account-home}/.config/dunst/dunstrc",
"owner": "${machine:account}",
"mode": "0644",
"template": "files/dunstrc"
},
{
"id": "notifier",
"type": "service",
"unit": "dunst.service",
"scope": "user",
"user": "${machine:account}",
"reload-on": [
"configuration"
]
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/dunst-tools",
"binary": "dunst-tools",
"loads": [
"dunst-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
},
"emits": [
"action-chosen",
"started",
"paused"
],
"items": [
{
"collection": "contribution:node-display-session:config",
"name": "display-session",
"template": "files/i3/50-dunst.conf"
}
]
}
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json
Vendored
+236
View File
@@ -0,0 +1,236 @@
{
"module": "xterm",
"version": "1",
"moved": true,
"capabilities": [
"package-manager"
],
"requires": [
"x11-display"
],
"claims": [
{
"name": "node-terminal-emulator",
"scope": "node",
"serves": [
"open"
]
}
],
"tools": [
"xterm_font",
"xterm_colours"
],
"config": {
"face": {
"type": "string",
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
"pattern": "[A-Za-z0-9 ._-]{1,64}",
"default": "JetBrainsMono Nerd Font"
},
"size": {
"type": "number",
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
"minimum": 6,
"maximum": 72,
"predefined": "font-size",
"default": 12
},
"save-lines": {
"type": "number",
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
"minimum": 0,
"maximum": 1000000,
"default": 10000
},
"term-name": {
"type": "string",
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
"pattern": "[A-Za-z0-9._-]{1,64}",
"default": "xterm-256color"
},
"render-font": {
"type": "boolean",
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
"default": true
},
"bold-mode": {
"type": "boolean",
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
"default": false
},
"always-bold-mode": {
"type": "boolean",
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
"default": false
},
"allow-bold-fonts": {
"type": "boolean",
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
"default": true
},
"background": {
"type": "string",
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"foreground": {
"type": "string",
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a8a8a8"
},
"color0": {
"type": "string",
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"color1": {
"type": "string",
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#9e1828"
},
"color2": {
"type": "string",
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#008800"
},
"color3": {
"type": "string",
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#d2bb4b"
},
"color4": {
"type": "string",
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#414171"
},
"color5": {
"type": "string",
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#963c59"
},
"color6": {
"type": "string",
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#418179"
},
"color7": {
"type": "string",
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bebebe"
},
"color8": {
"type": "string",
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#666666"
},
"color9": {
"type": "string",
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bc5766"
},
"color10": {
"type": "string",
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#61a171"
},
"color11": {
"type": "string",
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#e7db52"
},
"color12": {
"type": "string",
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#5085af"
},
"color13": {
"type": "string",
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a97a99"
},
"color14": {
"type": "string",
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#6ba4a4"
},
"color15": {
"type": "string",
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#ffffff"
},
"copy-binding": {
"type": "string",
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> C"
},
"paste-binding": {
"type": "string",
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> V"
}
},
"items": [
{
"collection": "environment",
"name": "terminal",
"attributes": {
"kind": "variable",
"name": "TERMINAL"
},
"template": "files/env-terminal"
},
{
"collection": "shell:xresources:normal",
"name": "xresources-normal",
"template": "files/xresources-normal"
}
],
"resources": [
{
"id": "package",
"type": "package",
"package": "xterm"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/xterm-tools",
"binary": "xterm-tools",
"loads": [
"xterm-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json
Vendored
+16
View File
@@ -0,0 +1,16 @@
{
"module": "mesh-sdk",
"version": "1",
"slug": "sdk",
"build": {
"artifacts": [
{
"name": "lib",
"kind": "package",
"language": "typescript",
"paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"]
}
]
},
"resources": []
}
modules/mesh-delivery-indexer/internal/index/toolchains.go
+44 -1
View File
@@ -134,6 +134,9 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWit
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is.
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error {
if err := ix.fillPaths(ctx); err != nil {
return err
}
if err := ix.recomputeModules(ctx, overlay); err != nil {
return err
}
@@ -366,7 +369,7 @@ func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Man
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.ParseManifest([]byte(kept.JSON))
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
}
// hasContext is whether an artifact of the manifest reads another repository's tree.
@@ -420,3 +423,43 @@ func (ix *Indexer) RecomputeForBuild(ctx context.Context) error {
}
return ix.Recompute(ctx)
}
// fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its
// newest build record names, once, by compare-and-set, changing nothing else (issue 1614): the builder needs it, and a
// rarely merged repository would leave it without one for weeks. A key that names a directory is never written here.
// This read of the controller's build records goes in R4g with build-agent.
func (ix *Indexer) fillPaths(ctx context.Context) error {
if ix.Modules == nil || ix.Builds == nil {
return nil
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
for _, key := range slices.Sorted(maps.Keys(held)) {
if held[key].Path != "" {
continue
}
dir, ok := ix.Builds.PathOf(key, held[key].Repository)
if !ok {
continue
}
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
if err != nil || cur.Path != "" {
return nil, false, nil
}
cur.Path = dir
data, err := delivery.EncodeModule(cur)
return data, false, err
})
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
ix.logf("modules.%s names no directory: %q filled from its newest build record (issue 1614)", key, dir)
}
return nil
}
modules/mesh-delivery-indexer/manifest_test.go
+8 -2
View File
@@ -38,6 +38,7 @@ type manifest struct {
Consumes []string `json:"consumes"`
State []state `json:"state"`
Reads []string `json:"reads"`
Invokes []string `json:"invokes"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
@@ -93,8 +94,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if m.SecretsOwner != "mesh-delivery-indexer" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// No invokes, no emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
for _, k := range []string{"invokes", "streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
// One invoke: the forge's trees with the named paths' objects, since the indexer is the one reader of a manifest's paths
// (ADR 0499 clauses 2 and 3, ADR 0495 clause 1, the lead's decision L8 of R4e).
if want := []string{"gitea.gitea_commit_trees"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// No emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the indexer has none", k)
}
modules/mesh-delivery-indexer/module.json
+3
View File
@@ -20,6 +20,9 @@
"mesh-delivery-checker.previews",
"mesh-delivery-gate.source"
],
"invokes": [
"gitea.gitea_commit_trees"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go
+1 -1
View File
@@ -136,7 +136,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket} {
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket} {
go follow(ctx, js, bucket, model, logger)
}
select {
modules/mesh-delivery-view/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-view/go.sum
+2 -10
View File
@@ -1,13 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337 h1:cZyFnvDiBpnNx8ifSUG9WA61OHsn7PC2ml2s4eD7AkA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a h1:Vjk2Kq+g/Q86N8CCHq3OatJIJ0JSy6Kh980ekGxNLmE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go
+53
View File
@@ -0,0 +1,53 @@
package view
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// I5 (issue 1851): from R4e the indexer writes the new form alone, and a module's fingerprint is built by the builder's key
// of it; the page shows the indexer's fp2 against that key's state, or the reason it has none.
const fp2A = "src2:3333333333333333333333333333333333333333333333333333333333333333"
func builderKey(t *testing.T, module, fp2 string, b delivery.BuilderBuild) Entry {
t.Helper()
key, err := delivery.BuilderBuildKey(module, fp2)
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeBuilderBuild(b)
if err != nil {
t.Fatal(err)
}
return Entry{Bucket: delivery.BuilderBuildsBucket, Key: key, Value: data, At: t0}
}
func TestAFingerprintOfTheNewFormIsShownAgainstItsBuilderKey(t *testing.T) { // I5
m := New()
manifest := strings.Repeat("a", 64)
d := "sha256:" + strings.Repeat("2", 64)
fill(m,
modEntry(t, "a", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}),
builderKey(t, "a", fp2A, delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.mesh-delivery-builder", Manifest: manifest,
Artifacts: []delivery.BuiltArtifact{{Name: "program", Kind: "bundle", Digest: d, Reference: "artifact-store://a/program/blobs/" + d}}}),
modEntry(t, "f", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}),
builderKey(t, "f", fp2A, delivery.BuilderBuild{State: delivery.BuildFailed, By: "shanks.mesh-delivery-builder", Manifest: manifest, Error: "not stored: program: 507"}),
modEntry(t, "n", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}),
modEntry(t, "r", delivery.Module{None2: "toolchain not yet reported", Repository: "novox/mesh-delivery", Commit: hA}))
s := m.Snapshot()
if r := fpRow(t, s, "a"); r.State != FPBuilt || r.Indexer != fp2A {
t.Errorf("a = %+v, want built with the indexer's fp2", r)
}
if r := fpRow(t, s, "f"); r.State != FPBuildFailed || !strings.Contains(r.Reason, "not stored") {
t.Errorf("f = %+v, want build failed with its error", r)
}
if r := fpRow(t, s, "n"); r.State != FPNoKey || r.Indexer != fp2A {
t.Errorf("n = %+v, want no key", r)
}
if r := fpRow(t, s, "r"); r.State != FPNone || r.Reason != "toolchain not yet reported" {
t.Errorf("r = %+v, want none with the new form's reason", r)
}
}
modules/mesh-delivery-view/internal/view/serve.go
+4
View File
@@ -146,6 +146,10 @@ func fingerprintText(s Snapshot) [][]string {
what = f.Reason
case FPNoBuild:
what = "indexer " + f.Indexer + "; no successful trunk build recorded a fingerprint"
case FPBuildFailed, FPBuilding:
what = "indexer " + f.Indexer + "; builder: " + f.Reason
case FPNoKey:
what = "indexer " + f.Indexer + "; the builder holds no key of it"
}
out = append(out, []string{f.Module, f.State, what})
}
modules/mesh-delivery-view/internal/view/view.go
+58 -1
View File
@@ -101,6 +101,12 @@ const (
FPDiffers = "differs"
FPNone = "none"
FPNoBuild = "no build"
// The states of a fingerprint of the new form against the builder's key of it (I5, issue 1851): built (the key is done),
// build failed (Reason is its error), building (queued, claimed or building; Reason says what it waits for) and no key.
FPBuilt = "built"
FPBuildFailed = "build failed"
FPBuilding = "building"
FPNoKey = "no key"
)
// FingerprintRow compares the indexer's fingerprint of a module at a trunk merge with the source fingerprint its newest
@@ -147,6 +153,7 @@ type Model struct {
builds map[string]buildEntry // by <module>.build.<id>
groups map[string]groupEntry // the indexer's candidates, by group
modules map[string]moduleEntry // the indexer's modules, by module
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
asks map[string]askRecord // the gate's asks, by key
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
newest map[string]string // module -> id
@@ -209,6 +216,11 @@ type groupEntry struct {
err string
}
type keyEntry struct {
value delivery.BuilderBuild
err string
}
type moduleEntry struct {
value delivery.Module
err string
@@ -233,7 +245,7 @@ type nodeEntry struct {
// New is an empty model.
func New() *Model {
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, asks: map[string]askRecord{}, answers: map[string]string{}, newest: map[string]string{},
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, answers: map[string]string{}, newest: map[string]string{},
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
}
@@ -321,6 +333,17 @@ func (m *Model) Apply(e Entry) {
id, _, _ = strings.Cut(a.Warrant, "@")
}
m.answers[e.Key] = id
case delivery.BuilderBuildsBucket:
if e.Deleted {
delete(m.keys, e.Key)
return
}
b, err := delivery.DecodeBuilderBuild(e.Value)
if err != nil {
m.keys[e.Key] = keyEntry{err: err.Error()}
return
}
m.keys[e.Key] = keyEntry{value: b}
case BuildsBucket:
module, rest, ok := strings.Cut(e.Key, ".")
if !ok {
@@ -525,6 +548,11 @@ func (m *Model) fingerprintRows() []FingerprintRow {
continue
}
row := FingerprintRow{Module: name, Repository: mod.value.Repository, Commit: mod.value.Commit}
if mod.value.FP2 != "" || mod.value.None2 != "" {
// The new form (R4e): the indexer's fp2 against the builder's key of it.
out = append(out, m.newFormRow(name, row, mod.value))
continue
}
b, hasBuild := newest[name]
switch {
case mod.value.None != "":
@@ -542,6 +570,35 @@ func (m *Model) fingerprintRows() []FingerprintRow {
return out
}
// newFormRow is a module's row of the new form: none with its reason, or its fp2 against the state of the builder's key
// of it (I5, issue 1851; the build records name no fingerprint of the new form).
func (m *Model) newFormRow(name string, row FingerprintRow, v delivery.Module) FingerprintRow {
if v.FP2 == "" {
row.State, row.Reason = FPNone, v.None2
return row
}
row.Indexer = v.FP2
key, err := delivery.BuilderBuildKey(name, v.FP2)
if err != nil {
row.State, row.Reason = FPNoKey, err.Error()
return row
}
k, ok := m.keys[key]
switch {
case !ok:
row.State = FPNoKey
case k.err != "":
row.State, row.Reason = FPBuildFailed, "the key cannot be read: "+k.err
case k.value.State == delivery.BuildDone:
row.State, row.Build = FPBuilt, v.FP2
case k.value.State == delivery.BuildFailed:
row.State, row.Reason = FPBuildFailed, k.value.Error
default:
row.State, row.Reason = FPBuilding, strings.TrimSpace(k.value.State+" "+k.value.Waits)
}
return row
}
// count is the number of items of a carried JSON array or object, and 0 for anything else.
func count(raw json.RawMessage) int {
var list []json.RawMessage