diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go index d92318e..b456fbd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go @@ -155,5 +155,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue. if err != nil { return catalogue.Manifest{}, err } - return catalogue.ParseManifest([]byte(kept.JSON)) + return catalogue.ParseManifestForInputs([]byte(kept.JSON)) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 6dd6b42..5ae38b7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -54,10 +54,12 @@ type fakeBuildModules struct { ctx context.Context n atomic.Int64 failBuild atomic.Bool - mu sync.Mutex - calls map[string]int // by `.` - fetches []map[string]any - builds []map[string]any // the arguments of every build call + // refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2). + refusePush atomic.Bool + mu sync.Mutex + calls map[string]int // by `.` + fetches []map[string]any + builds []map[string]any // the arguments of every build call } func (f *fakeBuildModules) count(tool string) int { @@ -119,8 +121,16 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { merge = `,"merge":"` + strings.Repeat("9", 40) + `"` } step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`) - case f.failBuild.Load() && node == "n1": + case f.failBuild.Load() && tool != "build_image_copy": // every node's build fails: one node builds a fingerprint step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"]) + case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4) + d := digestOf(fmt.Sprint(args["artifact"])) + step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"artifact":"`+fmt.Sprint(args["artifact"])+`","digest":"`+d+`","reference":"registry.example/`+fmt.Sprint(args["module"])+`/`+fmt.Sprint(args["artifact"])+`@`+d+`","pushed":true}`) + case tool != "build_image_build" && f.refusePush.Load(): + step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"]) + case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1) + d := digestOf(fmt.Sprint(args["artifact"])) + step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"artifact-store://`+filepath.Base(fmt.Sprint(args["module"]))+`/`+fmt.Sprint(args["artifact"])+`/blobs/`+d+`"}`) case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference d := digestOf(fmt.Sprint(args["artifact"])) step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"registry.example/imaged/server@`+d+`"}`) @@ -364,7 +374,7 @@ func isState(state string) func(delivery.BuilderBuild) bool { // TestBuilderMakesADoneKeyWithTheDigests: modules.netcheck (a Go and a TypeScript bundle) and its manifest are in the // indexer's buckets, and two nodes report each build module's toolchain; the program writes builds.netcheck. done, -// the digests of both bundles, both build modules' toolchains, after a fetch and a build on each of the two nodes. +// the digests of both bundles, both build modules' toolchains, after a fetch and a build on one node (ADR 0478 clause 4). func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b := startBed(t) digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") @@ -374,16 +384,16 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Fatalf("build = %+v", v) } for _, a := range v.Artifacts { - if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "" { - t.Errorf("artifact = %+v, want the digest every node built and no reference", a) + if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "artifact-store://netcheck/"+a.Name+"/blobs/"+a.Digest { + t.Errorf("artifact = %+v, want the digest every node built and the reference its push answered (ADR 0498 clause 1)", a) } } if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } - for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 2, "mesh-build-go.build_go_build": 2, "mesh-build-node.build_node_fetch": 2, "mesh-build-node.build_node_build": 2} { + for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} { if got := b.fake.count(tool); got != want { - t.Errorf("%s called %d times, want %d (once on each of two nodes)", tool, got, want) + t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) } } } @@ -400,8 +410,8 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // if len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files") { t.Fatalf("build = %+v, want the archive's digest", v) } - if n := b.fake.count("mesh-build-go.build_go_build"); n < 2 || n%2 != 0 { // each instance that built, on two nodes - t.Errorf("build_go_build was called %d times, want two for each instance that built", n) + if n := b.fake.count("mesh-build-go.build_go_build"); n < 1 { // each instance that built, on one node + t.Errorf("build_go_build was called %d times, want one for each instance that built", n) } } @@ -413,32 +423,7 @@ func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) { b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(delivery.BuildDone)) if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") { - t.Errorf("build = %+v, want done on two nodes naming no toolchain", v) - } -} - -// TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline: n2 is offline, so the key is done on n1 alone and marked; -// n2 coming online makes the builder build it again on both nodes, as a new revision (ADR 0478 clause 4). -func TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline(t *testing.T) { - b := startBed(t) - b.setOnline(t, "n2", false) - b.archive(t, "ca-trust", "modules/ca-trust", commit) - b.start(t, "n1", nil) - one := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if !strings.HasPrefix(one.Step, "one node (") { - t.Fatalf("build = %+v, want it marked as built on one node", one) - } - key := mustKey(t, "ca-trust") - e, err := b.builds.Get(b.ctx, key) - if err != nil { - t.Fatal(err) - } - b.setOnline(t, "n2", true) - two := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool { - return v.State == delivery.BuildDone && !strings.HasPrefix(v.Step, "one node (") - }) - if e2, _ := b.builds.Get(b.ctx, key); e2.Revision() <= e.Revision() || !strings.Contains(two.Step, "n1 and n2") { - t.Errorf("build = %+v at revision %d (was %d), want a new revision built on n1 and n2", two, e2.Revision(), e.Revision()) + t.Errorf("build = %+v, want done naming no toolchain", v) } } @@ -449,7 +434,7 @@ func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) { b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if !strings.Contains(v.Step, "on one node, n1") { + if !strings.Contains(v.Step, "on n1") { t.Errorf("build = %+v, want it built on n1 alone", v) } } @@ -463,7 +448,7 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFing b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(delivery.BuildFailed)) - if !strings.Contains(v.Error, "n1") || !strings.Contains(v.Error, "boom") { + if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") { t.Errorf("build = %+v, want failed naming the node and the step's reason", v) } key, _ := delivery.BuilderBuildKey("ca-trust", fp) @@ -500,8 +485,8 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil { t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2") } - if n := b.fake.count("mesh-build-go.build_go_build"); n != 2 { - t.Errorf("build_go_build was called %d times, want 2 (the new fingerprint on two nodes only)", n) + if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 { + t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n) } } @@ -674,3 +659,57 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 t.Errorf("builds = %v, want three, two of them with publish (the trunk's two fingerprints)", b.fake.builds) } } + +// TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain: with the artifact store refusing pushes, the key of an +// archive module ends failed opening "not stored:"; once it takes them, a new modules value builds the key again, done, its +// every artifact naming a reference (issue 1587, B8, ADR 0498 clauses 1, 2 and 8). +func TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain(t *testing.T) { + b := startBed(t) + b.fake.refusePush.Store(true) + b.archive(t, "ca-trust", "modules/ca-trust", commit) + b.start(t, "n1", nil) + v := b.await(t, "ca-trust", isState(delivery.BuildFailed)) + if !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") { + t.Fatalf("build = %+v, want failed opening %q with the store's words", v, delivery.NotStored) + } + b.fake.refusePush.Store(false) + b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef") + v = b.await(t, "ca-trust", isState(delivery.BuildDone)) + if ok, missing := v.Referenced(); !ok { + t.Errorf("build = %+v, artifacts naming no reference: %v", v, missing) + } +} + +// TestAPinnedUpstreamIsCopiedByTheImageBuildModule: hello-web's manifest (captured from mesh-catalog 49caf17f) names an +// upstream image pinned by its digest and an archive; the upstream is copied on n1, where mesh-build-image runs, with no +// fetch of the image module, and the key ends done with both references (issue 1587, B8, ADR 0498 clause 4). +func TestAPinnedUpstreamIsCopiedByTheImageBuildModule(t *testing.T) { + b := startBed(t) + b.moduleCommit(t, "hello-web", "catalogue-49caf17f-hello-web.module.json", "modules/hello-web", nil, commit) + b.start(t, "n1", nil) + v := b.await(t, "hello-web", isState(delivery.BuildDone)) + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, artifacts naming no reference: %v", v, missing) + } + if n := b.fake.count("mesh-build-image.build_image_copy"); n != 1 { + t.Errorf("build_image_copy called %d times, want once", n) + } + if n := b.fake.count("mesh-build-image.build_image_fetch"); n != 0 { + t.Errorf("build_image_fetch called %d times, want none for a copy", n) + } +} + +// TestAManifestFallingBackToAPredefinedKeyIsBuilt: claude-code's manifest (mesh-catalog 835bf22b) has a config value falling +// back to a predefined key; the builder, which holds no published list of them, reads it and builds the key done (issue +// 1751, mesh-sdk catalogue.ParseManifestForInputs). +func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) { + b := startBed(t) + b.module(t, "claude-code", "catalogue-835bf22b-claude-code.module.json", "modules/claude-code") + b.start(t, "n1", nil) + v := b.await(t, "claude-code", func(v delivery.BuilderBuild) bool { + return v.State == delivery.BuildDone || v.State == delivery.BuildFailed + }) + if v.State != delivery.BuildDone { + t.Errorf("build = %+v, want done: the manifest is read", v) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f2b61e6..40b6e29 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -35,8 +35,6 @@ const ( // defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a // build compiles). defaultStepBound = 30 * time.Minute - // nodesPerFingerprint is how many build nodes build each fingerprint until release R4e (ADR 0478 clause 4). - nodesPerFingerprint = 2 ) func main() { @@ -178,7 +176,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Health: have, Places: &buildrun.Places{Capacity: capacity}, Manifests: manifestReader{manifests}, - Nodes: nodesPerFingerprint, Logf: logger.Printf, } // The toolchains first, so a build that starts from the first modules value finds the nodes. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json new file mode 100644 index 0000000..005578c --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-49caf17f-hello-web.module.json @@ -0,0 +1,82 @@ +{ + "module": "hello-web", + "slug": "hello", + "version": "1", + "capabilities": [ + "container-runtime" + ], + "requires": [ + "route" + ], + "contributes": { + "route": { + "label": "hello", + "endpoint": "web" + } + }, + "binds": { + "route": "${dir:state}/route.json" + }, + "listens": [ + { + "name": "web", + "port": 8080, + "protocol": "tcp", + "from": "mesh", + "why": "the demo web page; only the route-proxy reaches it, and the public name is a route grant" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "page", + "type": "file", + "path": "${dir:state}/index.html", + "mode": "0644", + "template": "files/index.html" + }, + { + "id": "net", + "type": "network", + "name": "hello-web" + }, + { + "id": "server", + "type": "container", + "name": "hello-web", + "network": "hello-web", + "ports": [ + "8080" + ], + "volumes": [ + "${dir:state}/index.html:/www/index.html:ro" + ], + "args": [ + "sh", + "-c", + "while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done" + ], + "artifact": "server" + } + ], + "build": { + "artifacts": [ + { + "name": "server", + "kind": "upstream", + "from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json new file mode 100644 index 0000000..3b62d21 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/testdata/catalogue-835bf22b-claude-code.module.json @@ -0,0 +1,249 @@ +{ + "module": "claude-code", + "version": "1", + "moved": true, + "slug": "agent", + "capabilities": [ + "package-manager" + ], + "requires": [ + "mcp-endpoint", + { + "name": "gitea-read", + "optional": true + } + ], + "binds": { + "mcp-endpoint": "${dir:state}/mcp-endpoint.json" + }, + "secrets": { + "gitea-read": "${dir:state}/gitea-read.secret" + }, + "uses": [ + "operator-channel" + ], + "state": [ + "servers", + "holdings", + "config", + "proposals", + "questions", + "memory", + "waits" + ], + "reads": [ + "claude-licence-manager.bindings" + ], + "tools": [ + "claude_code_status", + "claude_code_render", + "claude_code_guard", + "claude_code_pull", + "claude_code_grant", + "claude_code_add_api_key", + "claude_code_registrations", + "claude_code_memory", + "claude_code_mcp_list", + "claude_code_mcp_register", + "claude_code_mcp_unregister", + "claude_code_skill_list", + "claude_code_skill_register", + "claude_code_skill_unregister", + "claude_code_agent_list", + "claude_code_agent_register", + "claude_code_agent_unregister", + "claude_code_command_list", + "claude_code_command_register", + "claude_code_command_unregister", + "claude_code_hook_list", + "claude_code_hook_register", + "claude_code_hook_unregister", + "claude_code_output_style_list", + "claude_code_output_style_register", + "claude_code_output_style_unregister", + "claude_code_instructions_list", + "claude_code_instructions_register", + "claude_code_instructions_unregister", + "claude_code_instructions_propose", + "claude_code_proposals", + "claude_code_ask", + "claude_code_answer", + "claude_code_wait", + "claude_code_settings_get", + "claude_code_settings_set", + "claude_code_settings_clear", + "claude_code_permission_add", + "claude_code_permission_remove", + "claude_code_config_list", + "claude_code_config_show", + "claude_code_config_status", + "claude_code_config_import", + "claude_code_home_show", + "claude_code_home_remove", + "claude_code_home_removed", + "claude_code_home_restore", + "claude_code_judge", + "claude_code_judged" + ], + "data": { + "own": [ + { + "id": "agent-home", + "path": "${dir:agent-home}", + "class": "valuable", + "why": "the operator's agent's own files: its memory, history and projects" + } + ] + }, + "config": { + "role": { + "type": "string", + "description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.", + "trusted": true, + "default": "" + }, + "mcp_servers": { + "type": "object", + "description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.", + "trusted": true, + "default": {} + }, + "managed_settings": { + "type": "object", + "description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.", + "trusted": true, + "predefined": "agent-managed-settings", + "default": {} + }, + "instructions": { + "type": "object", + "description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "output_styles": { + "type": "object", + "description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "skills": { + "type": "object", + "description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.", + "trusted": true, + "default": {} + }, + "commands": { + "type": "object", + "description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "agents": { + "type": "object", + "description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + } + }, + "resources": [ + { + "id": "package", + "type": "package", + "package": "claude-code" + }, + { + "id": "managed", + "type": "directory", + "path": "/etc/claude-code", + "mode": "0755", + "unknown": "program" + }, + { + "id": "start", + "type": "file", + "path": "/usr/local/bin/claude-agent", + "mode": "0755", + "template": "files/claude-agent" + }, + { + "id": "agent-account-name", + "type": "file", + "path": "/etc/claude-code/agent-account", + "mode": "0644", + "template": "files/agent-account" + }, + { + "id": "agent-account", + "type": "user", + "name": "${machine:agent-account}", + "home": "${machine:agent-home}", + "root": "${machine:agent-root}" + }, + { + "id": "agent-home", + "type": "directory", + "path": "${machine:agent-home}/.claude", + "mode": "0700", + "owner": "${machine:agent-account}", + "unknown": "module" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "${machine:account}", + "place": ".", + "unknown": "program" + }, + { + "id": "facts", + "type": "file", + "path": "${dir:state}/facts.json", + "mode": "0600", + "owner": "${machine:account}", + "template": "files/facts.json" + }, + { + "id": "settings", + "type": "file", + "path": "${dir:state}/settings.json", + "mode": "0600", + "owner": "${machine:account}", + "takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"], + "merge": "json", + "template": "files/settings.json" + } + ], + "items": [ + { + "collection": "shell:zsh:normal", + "template": "files/zsh-normal" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/claude-code", + "binary": "claude-code", + "loads": [ + "claude-code" + ], + "env": { + "MESH_CLAUDE_CODE_STATE": "${dir:state}", + "MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json", + "MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 59b27dd..284143f 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 3515fb8..db1ea18 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index 93e7183..d96df2c 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -69,6 +69,10 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) { } m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path, Toolchains: b.agreedFor(ctx, cm.Manifest)} + if len(cm.Paths) > 0 { + // The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4). + m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths} + } b.mu.Lock() if _, onTrunk := b.trunkFP[key]; !onTrunk { b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit} diff --git a/modules/mesh-delivery-builder/internal/build/candidates_test.go b/modules/mesh-delivery-builder/internal/build/candidates_test.go index 7c26902..97ff67a 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates_test.go +++ b/modules/mesh-delivery-builder/internal/build/candidates_test.go @@ -54,8 +54,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe t.Fatalf("build = %+v, want the candidate's fingerprint built", v) } fetches := r.bm.callsOf("build_go_fetch") - if len(fetches) != 2 { // one on each of two nodes - t.Fatalf("%d fetches, want one per node", len(fetches)) + if len(fetches) != 1 { // one node per fingerprint (ADR 0478 clause 4) + t.Fatalf("%d fetches, want one", len(fetches)) } f := fetches[0].body.(map[string]any) if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" { @@ -67,7 +67,7 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe // Two heads carrying one fingerprint are one build. r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY)) r.b.Wait() - if n := len(r.bm.callsOf("build_go_fetch")); n != 2 { + if n := len(r.bm.callsOf("build_go_fetch")); n != 1 { t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n) } } diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index cb5f74c..21b1316 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given. @@ -37,48 +39,71 @@ type Claim struct { Build delivery.BuilderBuild } -// OneNodePrefix opens the Step of a done build that a build module built on one node only, because no second node -// qualified (ADR 0478 clause 4); the build modules that did so follow in parentheses, then ": ". Claim reads it to build -// the key again, as a new revision, once a second node qualifies. -const OneNodePrefix = "one node (" - -// OneNodeModules are the build modules a done build built on one node only, or none when it is not such a build. -func OneNodeModules(b delivery.BuilderBuild) []string { - if b.State != delivery.BuildDone || !strings.HasPrefix(b.Step, OneNodePrefix) { - return nil - } - list, _, ok := strings.Cut(strings.TrimPrefix(b.Step, OneNodePrefix), "): ") - if !ok || list == "" { - return nil - } - return strings.Split(list, ", ") -} - // ClaimOptions say what the builder knows about the moment of a claim. type ClaimOptions struct { // TakeOver takes a key found building whoever holds it: the builder has just started and looks at the key for the first // time, and a build left building by a builder that is gone, on this node or another, is nobody's (ADR 0478 clause 6). TakeOver bool - // MoreNodes says whether one of these build modules, which built a done key on one node only, has a second node that - // qualifies now and the key may be built again (it is not held back after a failed second build, issue 1664). Nil means - // none does. - MoreNodes func(modules []string) bool // ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken // again only then. Nil means none does. ImageBuilder func() bool - // Trunk is whether the trunk's record names this fingerprint. A done key whose image has no reference was built as a - // candidate's, which never pushes (ADR 0487 clause 12); on the trunk it is taken again to publish the image, when an image - // builder runs (ImageBuilder). + // Trunk is whether the trunk's record names this fingerprint. A done key whose image or package has no reference was + // built as a candidate's, which never pushes or publishes (ADR 0487 clause 12, ADR 0498 clauses 5 and 6); on the trunk it + // is taken again to publish them, an image only when an image builder runs (ImageBuilder). Trunk bool + // Published says whether a package version `@` is published: one of the builder's done keys names it as + // a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500 + // clause 6). Nil means none is. + Published func(ref string) bool } -// Unpublished is whether a done build holds an image built without a push: it has no reference. -func Unpublished(b delivery.BuilderBuild) bool { +// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has +// no reference (a candidate's, ADR 0487 clause 12, ADR 0498 clauses 5 and 6). +func Unpublished(b delivery.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 } + +// UnpublishedModules are the build modules that publish what a done build holds unpublished: the image build module for +// an image, the Node build module for a package; sorted, none when it holds nothing unpublished. +func UnpublishedModules(b delivery.BuilderBuild) []string { + if b.State != delivery.BuildDone { + return nil + } + var image, pkg bool + for _, a := range b.Artifacts { + image = image || (a.Kind == catalogue.ArtifactImage && a.Reference == "") + pkg = pkg || (a.Kind == catalogue.ArtifactPackage && a.Reference == "") + } + var out []string + if image { + out = append(out, ImageBuildModule) + } + if pkg { + out = append(out, buildrun.NodeBuildModule) + } + sort.Strings(out) + return out +} + +// publishable are the build modules of UnpublishedModules that can publish now: the image build module only while a node +// runs it (imageBuilder). +func publishable(b delivery.BuilderBuild, imageBuilder func() bool) []string { + var out []string + for _, bm := range UnpublishedModules(b) { + if bm != ImageBuildModule || (imageBuilder != nil && imageBuilder()) { + out = append(out, bm) + } + } + return out +} + +// Unreferenced is whether a done build holds a bundle, an archive or an upstream copy that names no reference: a key done +// before R4e, whose artifacts stayed on their build nodes (ADR 0498 clause 8). An artifact of no kind (a key written before +// kinds) is one too. Claim takes such a key again, once: its new revision names them. +func Unreferenced(b delivery.BuilderBuild) bool { if b.State != delivery.BuildDone { return false } for _, a := range b.Artifacts { - if a.Kind == "image" && a.Reference == "" { + if a.Reference == "" && (a.Kind == "" || stored(a.Kind)) { return true } } @@ -87,13 +112,19 @@ func Unpublished(b delivery.BuilderBuild) bool { // Claim takes the build of module at fingerprint, or answers nil when it is not this builder's to build now: // - no key: it is created building, by this builder; a create that finds the key made meanwhile gets nothing; -// - a done key is not taken again (the same source is never built twice), except one built on one node only (OneNodePrefix) -// when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its -// builder writes the new revision when the second build ends (ADR 0478 clause 4); +// - a done key is not taken again (the same source is never built twice), except one whose bundle, archive or upstream +// copy names no reference (Unreferenced, a key done before R4e: ADR 0498 clause 8), which is claimed building as a new +// revision and built whole, and one whose image or package was built unpublished as a candidate's once the trunk +// names it (ClaimOptions.Trunk): that is taken as it stands, with the done value, and its builder writes the new +// revision when the publishing build ends (ADR 0498 clause 6). From R4e one node builds each fingerprint (ADR 0478 +// clause 4), so a key built on one node is never built again for a second; // - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no -// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), +// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478 +// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), // which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for // another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder); +// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is +// published (ClaimOptions.Published, ADR 0500 clause 6), and not before; // - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its // own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6). func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string, opt ClaimOptions) (*Claim, error) { @@ -115,19 +146,24 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string } switch { case cur.State == delivery.BuildDone: - if modules := OneNodeModules(cur); len(modules) > 0 && opt.MoreNodes != nil && opt.MoreNodes(modules) { - return &Claim{Key: key, Rev: rev, Build: cur}, nil + if Unreferenced(cur) { + return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) } - if opt.Trunk && Unpublished(cur) && opt.ImageBuilder != nil && opt.ImageBuilder() { + if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 { return &Claim{Key: key, Rev: rev, Build: cur}, nil } return nil, nil - case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): + case cur.State == delivery.BuildFailed && (delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): return nil, nil case cur.State == delivery.BuildFailed: return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) + case cur.State == delivery.BuildQueued && cur.Waits != "": + if opt.Published == nil || !opt.Published(cur.Waits) { + return nil, nil // the package it waits for is not published yet + } + return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) case cur.By == c.By || opt.TakeOver: return &Claim{Key: key, Rev: rev, Build: cur}, nil } diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index cab71e3..ed359a3 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -125,9 +125,9 @@ func TestClaimLosingTheCreateRaceGetsNothing(t *testing.T) { // 1607/AC3 } } -func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3 +func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3; a key whose artifacts name references (ADR 0498 clause 8) st := newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Digest: digest}}}) + seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/x/blobs/" + digest}}}) writes := st.writes got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}) if err != nil || got != nil || st.writes != writes { @@ -148,11 +148,19 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { } st = newMem() - key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: RefusedPrefix + " image (artifact x): build-agent builds it"}) + key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: delivery.RefusedKind + "package of language \"python\" (artifact x)"}) writes := st.writes if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes { t.Errorf("a refused kind: claim = %v, %v, writes %d (was %d), want none and no write", got, err, st.writes, writes) } + for _, e := range []string{delivery.NotPinned + "up: nats:2 names no digest", delivery.VersionTaken + "@novox/mesh-sdk@0.1.15"} { // B3, B5: ADR 0498 clauses 4, 5 + st = newMem() + seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: e}) + writes = st.writes + if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes { + t.Errorf("%s: claim = %v, %v, writes %d (was %d), want none until the fingerprint changes", e, got, err, st.writes, writes) + } + } } func TestAKeyFailedForNoImageBuilderIsTakenAgainOnlyWhenANodeHasTheModule(t *testing.T) { // 1705b diff --git a/modules/mesh-delivery-builder/internal/build/followups_test.go b/modules/mesh-delivery-builder/internal/build/followups_test.go index 6128d0c..ce0efd6 100644 --- a/modules/mesh-delivery-builder/internal/build/followups_test.go +++ b/modules/mesh-delivery-builder/internal/build/followups_test.go @@ -8,43 +8,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// 1664: a done key built on one node is not built again, over and over, while the second node's build keeps failing. -func TestASecondNodeBuildThatFailsIsNotRetriedUntilTheNodesChange(t *testing.T) { // issue 1664 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies - first := r.consider(t, "modules/m") - r.bm.fails["ace/build_go_fetch"] = "ace is out of disk" - r.pres.Put("ace", presenceValue(t, true), false) - r.b.NodesChanged(context.Background()) // ace came online: one more try - r.b.Wait() - tries := len(r.bm.callsOf("build_go_fetch")) - if tries < 2 { - t.Fatalf("%d fetches: the second node was never tried", tries) - } - if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) { - t.Fatalf("the done key changed: %+v", v) - } - // Nothing about the nodes changed: a modules value, a Reconsider and a start of looking again build nothing. - for i := 0; i < 3; i++ { - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Reconsider(context.Background()) - r.b.Wait() - } - if n := len(r.bm.callsOf("build_go_fetch")); n != tries { - t.Errorf("%d fetches after looking again, was %d: the failed second build was retried in a loop", n, tries) - } - // A node changing (ace goes and comes back) is a reason to try again, and ace now builds. - delete(r.bm.fails, "ace/build_go_fetch") - r.pres.Put("ace", presenceValue(t, false), false) - r.b.NodesChanged(context.Background()) - r.pres.Put("ace", presenceValue(t, true), false) - r.b.NodesChanged(context.Background()) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") { - t.Errorf("build = %+v, want it built on both nodes once ace could", v) - } -} - // 1660: at start the builder closes a key left building for a fingerprint the module no longer has. func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *testing.T) { // issue 1660 r := newRig(t, goBundle) diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index dc1f6b7..8c83b36 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -5,16 +5,14 @@ package build import ( "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) -// RefusedPrefix opens the error of a build key that is failed for a kind no builder builds yet (ADR 0478 clause 2): such -// a key is not built again as a new revision. -const RefusedPrefix = "refused kind:" - // ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that // run it. It has no toolchain language; the builder finds its nodes from the node-engine's have. const ImageBuildModule = "mesh-build-image" @@ -31,6 +29,9 @@ type Task struct { Kind string Module string Language string + // From is, for an upstream artifact, the reference its manifest pins, which the image build module copies (ADR 0498 + // clause 4). + From string } // Plan is what a manifest asks the builder to build, by artifact name. @@ -54,25 +55,34 @@ func (p Plan) Modules() []string { // (the indexer gives an archive no language, so a fingerprint names no toolchain for it; ADR 0478 clause 1). func (p Plan) NeedsToolchain(module string) bool { for _, t := range p.Tasks { - if t.Module == module && t.Kind != catalogue.ArtifactArchive && t.Kind != catalogue.ArtifactImage { + if t.Module == module && t.Kind != catalogue.ArtifactArchive && t.Kind != catalogue.ArtifactImage && t.Kind != catalogue.ArtifactUpstream { return true } } return false } -// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) it has no build -// module for. +// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) no build module +// builds. It opens delivery.RefusedKind, so its key is not built again until the fingerprint changes. type Refused struct{ Artifact, What string } func (r *Refused) Error() string { - return fmt.Sprintf("%s %s (artifact %s): the builder builds Go and TypeScript bundles and archives; build-agent builds it", RefusedPrefix, r.What, r.Artifact) + return fmt.Sprintf("%s%s (artifact %s): no build module builds it; the builder builds Go and TypeScript bundles, archives, images, upstream copies and TypeScript packages", delivery.RefusedKind, r.What, r.Artifact) } -// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2): a Go bundle and an archive are built by the Go build -// module, an image by the image build module (ADR 0487), a TypeScript bundle by the Node one, each in the order of the artifacts' names. A manifest with an -// upstream or package artifact, or a bundle of another language, is refused as a whole, naming the first such artifact -// by name. A manifest with no artifact has an empty plan. +// NotPinned is the error of a plan with an upstream artifact whose reference names no digest (ADR 0498 clause 4): it opens +// delivery.NotPinned, so its key is not built again until the fingerprint changes. +type NotPinned struct{ Artifact, From string } + +func (n *NotPinned) Error() string { + return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", delivery.NotPinned, n.Artifact, n.From) +} + +// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2, ADR 0498 clauses 4 and 5): a Go bundle and an archive are +// built by the Go build module, an image by the image build module (ADR 0487), which also copies an upstream image pinned +// by its digest, and a TypeScript bundle or package by the Node one, each in the order of the artifacts' names. A manifest +// with an upstream artifact pinned by no digest, or a bundle or a package of another language, or a kind no build module +// builds, is refused as a whole, naming the first such artifact by name. A manifest with no artifact has an empty plan. func PlanOf(m catalogue.Manifest) (Plan, error) { if m.Build == nil { return Plan{}, nil @@ -90,8 +100,16 @@ func PlanOf(m catalogue.Manifest) (Plan, error) { p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) case a.Kind == catalogue.ArtifactImage: p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule}) + case a.Kind == catalogue.ArtifactUpstream && !strings.Contains(a.From, "@sha256:"): + return Plan{}, &NotPinned{Artifact: a.Name, From: a.From} + case a.Kind == catalogue.ArtifactUpstream: + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule, From: a.From}) + case a.Kind == catalogue.ArtifactPackage && a.Language == "typescript": + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) case a.Kind == catalogue.ArtifactBundle: return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)} + case a.Kind == catalogue.ArtifactPackage: + return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("package of language %q", a.Language)} default: return Plan{}, &Refused{Artifact: a.Name, What: a.Kind} } diff --git a/modules/mesh-delivery-builder/internal/build/plan_test.go b/modules/mesh-delivery-builder/internal/build/plan_test.go index d6b18b2..4a70662 100644 --- a/modules/mesh-delivery-builder/internal/build/plan_test.go +++ b/modules/mesh-delivery-builder/internal/build/plan_test.go @@ -2,6 +2,8 @@ package build import ( "errors" + + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "testing" @@ -41,29 +43,35 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 } } -func TestPlanRefusesUpstreamAndPackageByName(t *testing.T) { // 1607/AC2, ADR 0478 clause 2 - for _, c := range []struct { - artifact catalogue.Artifact - kind string - }{ - {catalogue.Artifact{Name: "up", Kind: "upstream"}, "upstream"}, - {catalogue.Artifact{Name: "pkg", Kind: "package", Language: "typescript"}, "package"}, - } { - m := manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, c.artifact) - p, err := PlanOf(m) - if err == nil { - t.Errorf("%s: plan %+v, want a refusal", c.kind, p) - continue - } - var r *Refused - if !asRefused(err, &r) || !strings.HasPrefix(err.Error(), RefusedPrefix) || !strings.Contains(err.Error(), c.kind) || !strings.Contains(err.Error(), "build-agent") { - t.Errorf("%s: error = %v, want it to open %q and name the kind and build-agent", c.kind, err, RefusedPrefix) - } +const pinned = "nats:2.11.17-alpine@sha256:3333333333333333333333333333333333333333333333333333333333333333" + +func TestPlanSendsAPinnedUpstreamToTheImageBuildModulesCopyAndRefusesAnUnpinnedOne(t *testing.T) { // B3, ADR 0498 clause 4 + p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned})) + if err != nil || len(p.Tasks) != 2 { + t.Fatalf("plan %+v, error %v", p, err) + } + if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != ImageBuildModule || up.From != pinned || up.Language != "" { + t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, ImageBuildModule) + } + if p.NeedsToolchain(ImageBuildModule) { + t.Error("a copy needs no toolchain") + } + _, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"})) + if err == nil || !strings.HasPrefix(err.Error(), delivery.NotPinned+"up") || !delivery.FailedUntilChanged(err.Error()) { + t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, delivery.NotPinned) + } +} + +func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 clause 5 + _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"})) + var r *Refused + if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), delivery.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") { + t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, delivery.RefusedKind) } } func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) { - if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), RefusedPrefix) { + if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), delivery.RefusedKind) { t.Errorf("a python bundle: error = %v, want a refusal", err) } p, err := PlanOf(manifestOf()) diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go new file mode 100644 index 0000000..1c53597 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -0,0 +1,385 @@ +package build + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait +// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4). + +func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3 + r := newRig(t, goBundle, archiveA) + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + v := valueOf(t, r.st, r.key) + if v.State != delivery.BuildDone { + t.Fatalf("build = %+v, want done", v) + } + if ok, missing := v.Referenced(); !ok { + t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts) + } + rev := r.st.revs[r.key] + r.bm.mu.Lock() + r.bm.calls = nil + r.bm.mu.Unlock() + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint + r.b.Wait() + if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev { + t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev) + } +} + +func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2 + r := newRig(t, goBundle) + r.bm.noRef["ace/tool"] = true + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "tool") { + t.Fatalf("build = %+v, want failed opening %q and naming the artifact", v, delivery.NotStored) + } + delete(r.bm.noRef, "ace/tool") + if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want a not-stored key built again as a new revision", v) + } +} + +func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1, ADR 0498 clause 2 + r := newRig(t, goBundle) + r.bm.fails["ace/build_go_build"] = delivery.NotStored + "tool: the artifact store answered 507 insufficient storage" + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") || !strings.Contains(v.Error, "ace") { + t.Fatalf("build = %+v, want failed opening %q with the store's words and the node", v, delivery.NotStored) + } +} + +func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key + r := newRig(t, goBundle) + r.consider(t, "modules/m") + for _, c := range r.bm.callsOf("build_go_build") { + if c.body.(map[string]any)["publish"] != true { + t.Errorf("trunk build args = %+v, want publish", c.body) + } + } + c := newRig(t, goBundle) + trunkModule(t, c) + c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + c.b.Wait() + for _, call := range c.bm.callsOf("build_go_build") { + if _, ok := call.body.(map[string]any)["publish"]; ok { + t.Errorf("candidate build args = %+v, want no publish", call.body) + } + } +} + +// putBuild writes a builder key's value as the store would hold it. +func putBuild(t *testing.T, st *memStore, key string, b delivery.BuilderBuild) uint64 { + t.Helper() + data, err := delivery.EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + st.mu.Lock() + defer st.mu.Unlock() + return st.put(key, data) +} + +func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { // B2, ADR 0498 clause 8 + r := newRig(t, goBundle, archiveA) + rev := putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "built mesh-build-go on ace and novox", + Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest}, {Name: "tool", Kind: "bundle", Digest: digest}}}) + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev { + t.Fatalf("build = %+v at revision %d (was %d), want built again as a new revision", v, r.st.revs[r.key], rev) + } + if ok, missing := v.Referenced(); !ok { + t.Fatalf("the new revision names no reference for %v", missing) + } + rev = r.st.revs[r.key] + r.b.tried = map[string]bool{} // a builder started anew looks again + r.consider(t, "modules/m") + if r.st.revs[r.key] != rev { + t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev) + } +} + +func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4 + up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest} + r := newRig(t, up) + r.b.Health = imageHave(t, "ace", "novox") + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + t.Fatalf("build = %+v, want done with the copy", v) + } + if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest { + t.Errorf("artifact = %+v, want the upstream's digest and the registry's reference", a) + } + copies := r.bm.callsOf("build_image_copy") + if len(copies) != 1 { + t.Fatalf("%d copies, want one on one node", len(copies)) + } + if args := copies[0].body.(map[string]any); args["module"] != "m" || args["artifact"] != "up" || args["from"] != up.From || len(args) != 3 { + t.Errorf("copy args = %+v, want the module's name, the artifact and the pinned reference", args) + } + if n := len(r.bm.callsOf("build_image_fetch")) + len(r.bm.callsOf("build_image_build")); n != 0 { + t.Errorf("%d fetch or build calls for a copy, want none", n) + } +} + +func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4 + r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) + r.b.Health = imageHave(t, "ace") + v := r.consider(t, "modules/m") + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, missing references %v", v, missing) + } + if f, b, c := len(r.bm.callsOf("build_image_fetch")), len(r.bm.callsOf("build_image_build")), len(r.bm.callsOf("build_image_copy")); f != 1 || b != 1 || c != 1 { + t.Errorf("fetch %d, build %d, copy %d; want one each", f, b, c) + } +} + +func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2 + r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) + r.b.Health = imageHave(t, "ace") + r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500" + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) { + t.Fatalf("build = %+v, want failed 'not stored:' and built again", v) + } + delete(r.bm.fails, "ace/build_image_copy") + if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want done on the new revision", v) + } +} + +var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"} + +func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4 + c := newRig(t, tsPackage) + c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, c) + c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + c.b.Wait() + v := valueOf(t, c.st, c.key) + if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" { + t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v) + } + for _, call := range c.bm.callsOf("build_node_build") { + if _, ok := call.body.(map[string]any)["publish"]; ok { + t.Errorf("a candidate's package build was asked to publish: %+v", call.body) + } + } + + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + v = r.consider(t, "modules/m") + if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" { + t.Fatalf("trunk build = %+v, want done naming the published version", v) + } + if n := len(r.bm.callsOf("build_node_build")); n == 0 { + t.Error("the package was not built by the Node build module") + } +} + +func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B5, ADR 0498 clause 5 + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + r.bm.fails["ace/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" + r.bm.fails["novox/build_node_build"] = r.bm.fails["ace/build_node_build"] + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) || !strings.Contains(v.Error, "0.1.15") { + t.Fatalf("build = %+v, want failed opening %q naming the version", v, delivery.VersionTaken) + } + writes := r.st.writes + r.consider(t, "modules/m") + if r.st.writes != writes { + t.Errorf("a version-taken key was built again (%d writes, was %d)", r.st.writes, writes) + } +} + +func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6 + r := newRig(t, tsPackage, tsBundle) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + t.Fatalf("candidate build = %+v", v) + } + rev := r.st.revs[r.key] + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint + r.b.Wait() + v := valueOf(t, r.st, r.key) + if r.st.revs[r.key] <= rev { + t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev) + } + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, missing references %v", v, missing) + } + rev = r.st.revs[r.key] + r.b.tried = map[string]bool{} + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if r.st.revs[r.key] != rev { + t.Error("a published package was built again: once only") + } +} + +func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6 + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + for _, n := range []string{"ace", "novox"} { + r.bm.fails[n+"/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" + } + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) { + t.Errorf("build = %+v, want failed opening %q", v, delivery.VersionTaken) + } +} + +// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref. +func sdkModule(t *testing.T, r *rig, ref string) []byte { + t.Helper() + r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{tsPackage}}} + r.bm.packages["lib"] = ref + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "8a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: "src2:" + hex64('5'), + Manifest: mdigest('b'), Path: ".", Toolchains: agreedToolchains}) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePublished(t *testing.T) { // B7, ADR 0500 clause 6 + const ref = "@novox/mesh-sdk@0.1.15" + r := newRig(t, tsBundle) + for _, n := range []string{"ace", "novox"} { + r.bm.fails[n+"/build_node_fetch"] = waitsFor + ref + ": the lock names a version the registry does not hold" + } + v := r.consider(t, "modules/m") + if v.State != delivery.BuildQueued || v.Waits != ref || v.Error != "" { + t.Fatalf("build = %+v, want queued, waiting for %s, not failed", v, ref) + } + delete(r.bm.fails, "ace/build_node_fetch") + delete(r.bm.fails, "novox/build_node_fetch") + writes := r.st.writes + r.b.tried = map[string]bool{} + r.consider(t, "modules/m") + if r.st.writes != writes { + t.Fatalf("a key waiting for an unpublished package was claimed (%d writes, was %d)", r.st.writes, writes) + } + r.b.Module(context.Background(), "sdk", sdkModule(t, r, ref)) // the package's trunk key is built and published + r.b.Wait() + if v = valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Waits != "" { + t.Errorf("build = %+v, want built once the package it waits for is published", v) + } +} + +func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *testing.T) { // B7: the scan at start + const ref = "@novox/mesh-sdk@0.1.15" + r := newRig(t, tsBundle) + putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildQueued, By: "novox.builder", Manifest: manifestD, Waits: ref, Step: "waits"}) + sdkKey, _ := delivery.BuilderBuildKey("sdk", "src2:"+hex64('5')) + putBuild(t, r.st, sdkKey, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: mdigest('b'), Step: "built", + Artifacts: []delivery.BuiltArtifact{{Name: "lib", Kind: "package", Digest: digest, Reference: ref}}}) + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + if err := r.b.Sweep(context.Background()); err != nil { + t.Fatal(err) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want built: the package it waits for was published before the builder started", v) + } +} + +func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *testing.T) { // the lead's interface (c), ADR 0499 clause 4 + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')} + tree, err := delivery.NamedTree(manifestD, paths) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".", + Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}}) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "m", data) + r.b.Wait() + f := r.bm.callsOf("build_node_fetch") + if len(f) == 0 { + t.Fatal("no fetch") + } + req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any)) + if err != nil || req.Dir != "." || strings.Join(req.Paths, ",") != "package.json,src" || req.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" { + t.Errorf("fetch request = %+v, %v; want dir . and the kept paths, sorted", req, err) + } + r2 := newRig(t, goBundle) + r2.consider(t, "modules/m") + if a := r2.bm.callsOf("build_go_fetch")[0].body.(map[string]any); a["dir"] != nil || a["paths"] != nil { + t.Errorf("fetch args = %+v, want no dir or paths for a module naming none", a) + } +} + +func hex40(c byte) string { return strings.Repeat(string(c), 40) } + +func TestOneNodeBuildsEachFingerprint(t *testing.T) { // B9, ADR 0478 clause 4: "From R4e one node builds each fingerprint" + r := newRig(t, goBundle, archiveA, tsBundle) // ace and novox both qualify for both build modules + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || strings.Contains(v.Step, " and ") || strings.HasPrefix(v.Step, "one node (") { + t.Fatalf("build = %+v, want done on one node of each build module, unmarked", v) + } + if f, b := len(r.bm.callsOf("build_go_fetch")), len(r.bm.callsOf("build_go_build")); f != 1 || b != 2 { + t.Errorf("go fetches %d, builds %d; want one fetch and one build of each artifact, on one node", f, b) + } + if f := len(r.bm.callsOf("build_node_fetch")); f != 1 { + t.Errorf("node fetches %d, want one", f) + } +} + +func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: the second-node retake is gone + r := newRig(t, goBundle) + putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, + Step: "one node (mesh-build-go): built mesh-build-go on one node, novox", + Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/tool/blobs/" + digest}}}) + writes := r.st.writes + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.NodesChanged(context.Background()) + r.b.Wait() + if r.st.writes != writes || len(r.bm.calls) != 0 { + t.Errorf("a key done on one node was built again (%d writes, was %d; %d calls)", r.st.writes, writes, len(r.bm.calls)) + } +} + +func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths + r := newRig(t, tsPackage) + r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" + trunkModule(t, r) + pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: headX}, Modules: []string{"m"}, + Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}} + hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) + data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + r.b.Candidate(context.Background(), "feat%2Fx", data) + r.b.Wait() + f := r.bm.callsOf("build_node_fetch") + if len(f) == 0 { + t.Fatal("no fetch") + } + req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any)) + if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" { + t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index 24f962f..f2bcc51 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -34,8 +34,8 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 { t.Fatalf("build = %+v, want done with the archive and no toolchain named", v) } - if n := len(r.bm.callsOf("build_go_build")); n != 2 { - t.Errorf("%d go builds, want one on each of two nodes", n) + if n := len(r.bm.callsOf("build_go_build")); n != 1 { + t.Errorf("%d go builds, want one, on one node (ADR 0478 clause 4)", n) } } @@ -54,8 +54,8 @@ func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *te r.b.Reconsider(context.Background()) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix) { - t.Errorf("build = %+v, want done on novox, the one node that qualifies, marked as one node", v) + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { + t.Errorf("build = %+v, want done on novox, the one node that qualifies", v) } } @@ -70,25 +70,6 @@ func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testi } } -// 2. The digests-differ failure opens with its prefix, and nothing builds it again. -func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testing.T) { // review round 4, 2 - r := newRig(t, goBundle) - r.bm.digests["ace/tool"] = "sha256:" + hex64('9') - v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, "digests differ:") { - t.Fatalf("build = %+v, want a failure opening with exactly %q", v, "digests differ:") - } - calls, writes := len(r.bm.calls), r.st.writes - r.bm.digests = map[string]string{} // the nodes would now agree - r.b.Reconsider(context.Background()) - r.b.Wait() - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if len(r.bm.calls) != calls || r.st.writes != writes { - t.Errorf("a failed-for-digests key was built again: %d calls (was %d), %d writes (was %d)", len(r.bm.calls), calls, r.st.writes, writes) - } -} - func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2 st := newMem() seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) @@ -99,67 +80,6 @@ func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review } } -// 3. A done key built on one node is built again on a second node once one qualifies. -func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *testing.T) { // review round 4, 3 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+buildrun.GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") { - t.Fatalf("build = %+v, want done on one node, marked", v) - } - if got := OneNodeModules(v); len(got) != 1 || got[0] != buildrun.GoBuildModule { - t.Errorf("OneNodeModules = %v", got) - } - rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build")) - // Nothing changes while one node qualifies: looking again builds nothing. - r.b.Reconsider(context.Background()) - r.b.Wait() - if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_go_build")) != calls { - t.Fatal("a one-node key was built again with no second node") - } - // ace comes online: the key is built on ace and novox, as a new revision. - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - v = valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") { - t.Errorf("build = %+v, want done on ace and novox", v) - } - if r.st.revs[r.key] <= rev { - t.Errorf("revision %d, was %d: not a new revision", r.st.revs[r.key], rev) - } - builds := r.bm.callsOf("build_go_build") - if len(builds) != calls+2 || builds[len(builds)-2].node == builds[len(builds)-1].node { - t.Errorf("%d builds (was %d), want one on each of two nodes", len(builds), calls) - } -} - -func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.T) { // review round 4, 3 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) - r.consider(t, "modules/m") - r.bm.digests["ace/tool"] = "sha256:" + hex64('9') - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, buildrun.DigestsDifferPrefix) { - t.Errorf("build = %+v, want failed with digests differ", v) - } -} - -func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testing.T) { // review round 4, 3 - r := newRig(t, goBundle) - r.pres.Put("ace", presenceValue(t, false), false) - first := r.consider(t, "modules/m") - r.bm.fails["ace/build_go_fetch"] = "ace is out of disk" - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) { - t.Errorf("build = %+v, want the first done build to stand: %+v", v, first) - } -} - // 4. A step never waits on a full node while another qualifying node has room. func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4 p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}} @@ -252,7 +172,7 @@ func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) r.b.Health = have v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on one node, novox") { + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { t.Fatalf("build = %+v, want done on novox alone", v) } for _, c := range r.bm.calls { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ea67174..9c6f23e 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -5,13 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "sort" "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). @@ -41,8 +42,6 @@ type Builder struct { } // buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node. Places *buildrun.Places - // Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4). - Nodes int // Limit is how many builds run at once; four when 0. Limit int Logf func(format string, args ...any) @@ -59,10 +58,9 @@ type Builder struct { via map[string]via // by build key: a candidate's fetch (candidates.go) trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint tried map[string]bool // by build key: looked at since this process started - // epoch counts the changes of the nodes (online, healthy, a toolchain) seen since this process started; a done key - // whose second-node build failed is held back until it moves (issue 1664). - epoch uint64 - secondAt map[string]uint64 // by build key: the epoch a second-node build of a one-node key failed at + // published are the package versions `@` a done key of this bucket names as a package's reference: a key + // waiting for one is built once it is here (ADR 0500 clause 6). + published map[string]bool } type planned struct { @@ -79,7 +77,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} - b.secondAt = map[string]uint64{} + b.published = map[string]bool{} if b.Places == nil { b.Places = &buildrun.Places{} } @@ -116,20 +114,14 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { } // NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or -// stopped being, or a toolchain moved. A done key built on one node whose second-node build failed is tried again only -// after such a change, not at every modules value (issue 1664). -func (b *Builder) NodesChanged(ctx context.Context) { - b.init() - b.mu.Lock() - b.epoch++ - b.mu.Unlock() - b.Reconsider(ctx) -} +// stopped being, or a toolchain moved, so a build that waited for a node may go on. +func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) } -// Sweep closes, at start, every key left building for a fingerprint its module no longer has (issue 1660, ADR 0478 -// clause 6): the key can never be built, since nothing asks for it, and would stay building for ever. It fails the key -// naming who left it. A key of the fingerprint the module has now is not touched: the build takes it over. Call it -// once the modules bucket has been read, so every module is known. +// Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key +// waiting for one is then looked at again, ADR 0500 clause 6), and closes every key left building for a fingerprint its +// module no longer has (issue 1660, ADR 0478 clause 6): the key can never be built, since nothing asks for it, and would +// stay building for ever. It fails the key naming who left it. A key of the fingerprint the module has now is not +// touched: the build takes it over. Call it once the modules bucket has been read, so every module is known. func (b *Builder) Sweep(ctx context.Context) error { b.init() keys, err := b.Claimer.Store.Keys(ctx) @@ -146,16 +138,18 @@ func (b *Builder) Sweep(ctx context.Context) error { } } b.mu.Unlock() + newly := false for _, key := range keys { - if current[key] { - continue - } value, rev, found, err := b.Claimer.Store.Get(ctx, key) if err != nil || !found { continue } cur, err := delivery.DecodeBuilderBuild(value) - if err != nil || cur.State != delivery.BuildBuilding { + if err != nil { + continue + } + newly = b.notePublished(cur) || newly + if current[key] || cur.State != delivery.BuildBuilding { continue } closed := delivery.BuilderBuild{State: delivery.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step, @@ -169,9 +163,36 @@ func (b *Builder) Sweep(ctx context.Context) error { } b.logf("%s: closed, left building by %s for a fingerprint the module no longer has", key, cur.By) } + if newly { + // A key read before the scan may wait for a package the scan found published (ADR 0500 clause 6). + b.Reconsider(ctx) + } return nil } +// notePublished keeps the package versions a done build names as references, and says whether one was new. +func (b *Builder) notePublished(v delivery.BuilderBuild) bool { + if v.State != delivery.BuildDone { + return false + } + b.mu.Lock() + defer b.mu.Unlock() + newly := false + for _, a := range v.Artifacts { + if a.Kind == catalogue.ArtifactPackage && a.Reference != "" && !b.published[a.Reference] { + b.published[a.Reference] = true + newly = true + } + } + return newly +} + +func (b *Builder) isPublished(ref string) bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.published[ref] +} + // Forget takes a module out (its key was deleted). func (b *Builder) Forget(key string) { b.init() @@ -275,10 +296,8 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module // building by this builder: it takes it again when it starts (ADR 0478 clause 5). func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { plan, planErr, read := b.planOf(ctx, m.Manifest) - if read && planErr == nil && len(plan.Tasks) == 0 { - return // nothing in the manifest for a builder to build - } - if planErr == nil { + nothing := read && planErr == nil && len(plan.Tasks) == 0 + if planErr == nil && !nothing { if why := b.stale(m, plan); why != "" { // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build // now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written. @@ -291,14 +310,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { trunk := b.trunkFP[key] first := !b.tried[key] b.tried[key] = true - epoch := b.epoch - failedAt, failedBefore := b.secondAt[key] b.mu.Unlock() - moreNodes := func(modules []string) bool { - // A second build that failed is not tried again until the nodes change (issue 1664). - return (!failedBefore || epoch > failedAt) && b.moreNodes(modules) - } - cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: moreNodes, ImageBuilder: b.imageBuilderRuns, Trunk: trunk}) + cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished}) if err != nil { b.logf("%s: %v", module, err) return @@ -306,22 +319,28 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if cl == nil { return } - // A done key taken again builds only what is missing: the build modules that ran on one node, and the image when it was - // built as a candidate's; the other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8). + // A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the + // other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6). var prior *delivery.BuilderBuild var only map[string]bool publishing := false if cl.Build.State == delivery.BuildDone { prior, only = &cl.Build, map[string]bool{} - for _, bm := range OneNodeModules(cl.Build) { - only[bm] = true - } - if trunk && Unpublished(cl.Build) { - only[ImageBuildModule] = true - publishing = true + if trunk { + for _, bm := range publishable(cl.Build, b.imageBuilderRuns) { + only[bm] = true + publishing = true + } } } - result := b.build(ctx, module, m, plan, planErr, prior, only) + var result delivery.BuilderBuild + if nothing { + // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as + // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). + result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"} + } else { + result = b.build(ctx, module, m, plan, planErr, prior, only) + } if ctx.Err() != nil { return } @@ -331,17 +350,9 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } result.By, result.Manifest = b.Claimer.By, m.Manifest if publishing && result.State == delivery.BuildFailed { - // The trunk's image was built only as a candidate's and its push failed: the key says so, so the builds view, the + // The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the // checker and the failed-key retake see it, and a look does not build the image again for nothing. - result.Error = "publishing the image of the merged commit failed: " + result.Error - } else if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, buildrun.DigestsDifferPrefix) { - // A done key built again on a second node keeps its done value when the new build fails for another reason than - // the digests: the first build stands, and a later change of the nodes looks again. - b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error) - b.mu.Lock() - b.secondAt[cl.Key] = epoch - b.mu.Unlock() - return + result.Error = keyError([]string{"publishing what the merged commit built failed: " + result.Error}) } if err := b.Claimer.Finish(ctx, cl, result); err != nil { b.logf("%s: the build could not be written: %v", cl.Key, err) @@ -350,6 +361,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if result.State == delivery.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } + if result.State == delivery.BuildQueued { + b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits) + } + if b.notePublished(result) { + // A key may wait for the package this build published (ADR 0500 clause 6). + b.Reconsider(ctx) + } } // stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan @@ -396,7 +414,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p repository := delivery.RepositoryName(m.Repository) var lanes []*lane used := map[string]delivery.ToolchainUsed{} - var steps, oneNode []string + var steps []string // Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up. giveUp := func(nodes ...string) { for _, n := range nodes { @@ -429,7 +447,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } nodes = ns } else if plan.NeedsToolchain(bm) { - ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, b.Nodes) + ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, 1) if err != nil { giveUpLanes() return failed("%s", err) @@ -443,23 +461,15 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p nodes = ns } else { // An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies. - ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes) + ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, 1) if err != nil { giveUpLanes() return failed("%s", err) } nodes = ns } - if bm == ImageBuildModule { - steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0])) - } else if len(nodes) == 1 { - oneNode = append(oneNode, bm) - steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0])) - } else { - sorted := append([]string(nil), nodes...) - sort.Strings(sorted) - steps = append(steps, fmt.Sprintf("%s on %s", bm, strings.Join(sorted, " and "))) - } + // One node builds each fingerprint (ADR 0478 clause 4, from R4e). + steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0])) var tasks []Task for _, t := range plan.Tasks { if t.Module == bm { @@ -483,7 +493,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } if len(errs) > 0 { - return failed("%s", strings.Join(errs, "; ")) + if ref := waitsOf(errs); ref != "" { + // The module's lock names a version of a package the registry does not hold yet: the key waits for it, queued, and + // is not failed (ADR 0500 clause 6). + return delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref, Step: "waits for the package " + ref + " to be published"} + } + return failed("%s", keyError(errs)) } var artifacts []delivery.BuiltArtifact keptModule := map[string]bool{} @@ -515,7 +530,16 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } built := delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d} for _, l := range lanes { - if r := l.references[t.Artifact]; r != "" && l.module == t.Module { + if l.module != t.Module { + continue + } + r := l.references[t.Artifact] + if r == "" && stored(t.Kind) { + // A bundle, an archive or an upstream copy is done only when its push was answered with where it is (ADR 0498 + // clause 1); a candidate's included (clause 3). + return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", delivery.NotStored, t.Artifact, l.node) + } + if r != "" { built.Reference = r } } @@ -530,14 +554,64 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } step := "built " + strings.Join(steps, "; ") - if len(oneNode) > 0 { - // Marked so that Claim builds the key again once a second node qualifies (ADR 0478 clause 4). - step = OneNodePrefix + strings.Join(oneNode, ", ") + "): " + step - } if len(used) == 0 { used = nil } - return delivery.BuilderBuild{State: delivery.BuildDone, Step: step, Artifacts: artifacts, Toolchains: used} + done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used} + if _, err := delivery.EncodeBuilderBuild(done); err != nil { + // A reference the build module answered that does not name the artifact and its digest is no reference. + return failed("%s%v", delivery.NotStored, err) + } + return done +} + +// stored is whether an artifact of this kind is in the artifact store once built, a candidate's too, so its done key +// names its reference (ADR 0498 clauses 1, 3 and 4); an image or a package names one only when built for the trunk. +func stored(kind string) bool { + return kind == catalogue.ArtifactBundle || kind == catalogue.ArtifactArchive || kind == catalogue.ArtifactUpstream +} + +// waitsFor opens the reason of a fetch step whose module's lock names a package version the registry does not hold, then +// `@` (the Node build module's, ADR 0500 clause 6). +const waitsFor = delivery.WaitsFor + +// waitsOf is the package version a lane's error says the module waits for, or "" when none does or what it names is not +// `@`. +func waitsOf(errs []string) string { + for _, e := range errs { + at := strings.Index(e, waitsFor) + if at < 0 { + continue + } + ref, _, _ := strings.Cut(e[at+len(waitsFor):], " ") + ref = strings.TrimRight(ref, ":;,") + if _, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref}); err == nil { + return ref + } + } + return "" +} + +// keyPrefixes are the errors a build module's step may open its reason with that the key's error must open with too, so +// the claim reads them (delivery.FailedUntilChanged), first the ones not built again. +var keyPrefixes = []string{delivery.NotPinned, delivery.VersionTaken, delivery.RefusedKind, delivery.NotStored} + +// keyError is the error of a key from its lanes' errors: when one names a prefix of keyPrefixes inside the step's words +// (` on : the step failed: not stored: ...`), the key's error opens with that prefix, so a reader of the +// key need not search for it. +func keyError(errs []string) string { + for _, prefix := range keyPrefixes { + for i, e := range errs { + if strings.HasPrefix(e, prefix) { + return strings.Join(errs, "; ") + } + if at := strings.Index(e, prefix); at >= 0 { + rest := append([]string{prefix + e[:at] + e[at+len(prefix):]}, append(append([]string(nil), errs[:i]...), errs[i+1:]...)...) + return strings.Join(rest, "; ") + } + } + } + return strings.Join(errs, "; ") } func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) } @@ -580,24 +654,6 @@ func (b *Builder) imageBuilderRuns() bool { return false } -// moreNodes is whether any of these build modules has two nodes that qualify now. -func (b *Builder) moreNodes(modules []string) bool { - for _, bm := range modules { - seen := map[string]bool{} - n := 0 - for _, e := range b.Toolchains.Entries(buildrun.LanguageOf[bm]) { - if !seen[e.Node] && b.qualifies(e.Node, bm) { - seen[e.Node] = true - n++ - } - } - if n >= 2 { - return true - } - } - return false -} - // step runs one step of a build module; the lane that runs it holds its node's place. func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) { return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args) @@ -616,6 +672,8 @@ type buildResult struct { } // runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time. +// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches +// nothing. func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) { release, err := b.Places.Acquire(ctx, l.node) if err != nil { @@ -623,57 +681,83 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } defer release() - fetchArgs := map[string]any{"repository": repository, "commit": m.Commit} + req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit} if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { b.mu.Lock() v, candidate := b.via[key] b.mu.Unlock() if candidate { - fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base + req.Ref, req.Base = v.Ref, v.Base } } - raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) - if err != nil { - l.err = err - return + if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 { + // A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from + // the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e). + req.Dir = m.Path + for p := range m.Inputs2.Paths { + req.Paths = append(req.Paths, p) + } + sort.Strings(req.Paths) } + fetchArgs := req.Args() + candidate := req.Base != "" var fetched fetchResult - if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { - l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) - return - } - // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at - // the trunk, and building that would store the trunk's build under the candidate's fingerprint. - _, candidate := fetchArgs["base"] - if candidate { - if len(fetched.Conflicts) > 0 { - l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + if needsFetch(l.tasks) { + raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) + if err != nil { + l.err = err return } - if fetched.Merge == "" { - l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { + l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) return } + // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at + // the trunk, and building that would store the trunk's build under the candidate's fingerprint. + if candidate { + if len(fetched.Conflicts) > 0 { + l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + return + } + if fetched.Merge == "" { + l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + return + } + } } for _, t := range l.tasks { - args := map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} - if t.Kind == catalogue.ArtifactImage && !candidate { - // Only a trunk key pushes the image to the registry; a candidate's build never does (ADR 0487 clause 12). + tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} + if t.Kind == catalogue.ArtifactUpstream { + // The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin. + tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From} + } else if !candidate { + // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, + // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). args["publish"] = true } - raw, err := b.step(ctx, l.module, l.node, "build", args) + raw, err := b.step(ctx, l.module, l.node, tool, args) if err != nil { l.err = err return } var built buildResult if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" { - l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw) + l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw) return } l.digests[t.Artifact] = built.Digest - if built.Reference != "" && !candidate { + if built.Reference != "" && (stored(t.Kind) || !candidate) { l.references[t.Artifact] = built.Reference } } } + +// needsFetch is whether any of the tasks builds from the module's source: every one but an upstream copy. +func needsFetch(tasks []Task) bool { + for _, t := range tasks { + if t.Kind != catalogue.ArtifactUpstream { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 6da1ebe..4a8524c 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -23,6 +23,10 @@ type buildModules struct { steps *fakeSteps digests map[string]string // `/` -> digest; the default is `digest` fails map[string]string // `/` -> the reason the step fails with + // noRef names `/` whose bundle or archive build answers no reference (its push not answered). + noRef map[string]bool + // packages are the package artifacts by name, each with the `@` its trunk build publishes (ADR 0498 clause 5). + packages map[string]string // hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives. // conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict. conflicts string @@ -32,7 +36,7 @@ type buildModules struct { } func newBuildModules() *buildModules { - return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}} + return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}} } func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) { @@ -67,8 +71,18 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok { d = v } - if tool == "build_image_build" && args["publish"] == true { - step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/img@` + d + `"}`) + if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4) + step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`) + } else if pkg, ok := b.packages[fmt.Sprint(args["artifact"])]; ok { // a package names its version only when published + step.Result = json.RawMessage(`{"digest":"` + d + `"}`) + if args["publish"] == true { + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"` + pkg + `"}`) + } + } else if tool == "build_image_build" && args["publish"] == true { + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`) + } else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] { + // A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1). + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`) } else { step.Result = json.RawMessage(`{"digest":"` + d + `"}`) } @@ -125,7 +139,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { } r.b = &Builder{ Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}, - Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf, + Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf, } var err error if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil { @@ -166,15 +180,15 @@ var ( manifestD = mdigest('a') ) -func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 +func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 r := newRig(t, goBundle, archiveA, tsBundle) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } for i, want := range []string{"files", "tool", "web"} { - if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "" { - t.Errorf("artifact %d = %+v, want %s with the digest and no reference (ADR 0478 clause 6)", i, a, want) + if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "artifact-store://m/"+want+"/blobs/"+digest { + t.Errorf("artifact %d = %+v, want %s with the digest and its artifact-store reference (ADR 0498 clause 1)", i, a, want) } } if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" { @@ -186,15 +200,15 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test if got := v.Toolchains["typescript"]; got.Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } - // Two fetches per build module, one on each node; every build names the module's directory. - if n := len(r.bm.callsOf("build_go_fetch")); n != 2 { - t.Errorf("%d go fetches, want one per node", n) + // One fetch per build module, on one node (ADR 0478 clause 4); every build names the module's directory. + if n := len(r.bm.callsOf("build_go_fetch")); n != 1 { + t.Errorf("%d go fetches, want one", n) } - if n := len(r.bm.callsOf("build_go_build")); n != 4 { // tool and files, each on two nodes - t.Errorf("%d go builds, want 4", n) + if n := len(r.bm.callsOf("build_go_build")); n != 2 { // tool and files + t.Errorf("%d go builds, want 2", n) } - if n := len(r.bm.callsOf("build_node_build")); n != 2 { - t.Errorf("%d node builds, want 2", n) + if n := len(r.bm.callsOf("build_node_build")); n != 1 { + t.Errorf("%d node builds, want 1", n) } c := r.bm.callsOf("build_go_build")[0] if args := c.body.(map[string]any); args["module"] != "modules/m" || !strings.HasPrefix(fmt.Sprint(args["workspace"]), "w-") { @@ -206,21 +220,12 @@ func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *test } } -func TestDifferingDigestsFailTheBuildNamingBothNodes(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 - r := newRig(t, goBundle) - r.bm.digests["ace/tool"] = "sha256:" + hex64('9') - v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "novox") || !strings.Contains(v.Error, digest) || !strings.Contains(v.Error, hex64('9')) { - t.Errorf("build = %+v, want failed naming both nodes and both digests", v) - } -} - -func TestOneQualifyingNodeBuildsOnceAndTheKeySaysSo(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 +func TestOneQualifyingNodeBuildsOnce(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 r := newRig(t, goBundle) r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true) v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "one node") || !strings.Contains(v.Step, "novox") { - t.Errorf("build = %+v, want done with its step saying one node, novox", v) + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { + t.Errorf("build = %+v, want done with its step saying novox", v) } if n := len(r.bm.callsOf("build_go_build")); n != 1 { t.Errorf("%d builds, want one", n) @@ -263,9 +268,6 @@ func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { / if v.Artifacts[0].Reference == "" { t.Errorf("a trunk image has no reference: %+v", v.Artifacts[0]) } - if strings.HasPrefix(v.Step, OneNodePrefix) || len(OneNodeModules(v)) != 0 { - t.Errorf("step = %q, want no one-node mark: an image is never built again on a second node", v.Step) - } if n := len(r.bm.callsOf("build_image_fetch")); n != 1 { t.Errorf("%d image fetches, want one", n) } @@ -286,22 +288,19 @@ func TestAnImageIsBuiltOnlyWhereMeshBuildImageIsHealthy(t *testing.T) { // 1705a } } -func TestABundleAndAnImageOfOneModuleBuildOnTwoNodesAndOne(t *testing.T) { // 1705a +func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a r := newRig(t, goBundle, imageArt) r.b.Health = imageHave(t, "ace", "novox") v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) } - if n := len(r.bm.callsOf("build_go_build")); n != 2 { - t.Errorf("%d go builds, want 2", n) + if n := len(r.bm.callsOf("build_go_build")); n != 1 { + t.Errorf("%d go builds, want 1 (one node per fingerprint, ADR 0478 clause 4)", n) } if n := len(r.bm.callsOf("build_image_build")); n != 1 { t.Errorf("%d image builds, want 1", n) } - if modules := OneNodeModules(v); len(modules) != 0 { - t.Errorf("one-node modules = %v, want none", modules) - } } func TestWithNoImageNodeTheKeyFailsNamingTheProvisionAndIsNotRetakenUntilANodeReportsTheModule(t *testing.T) { // 1705b, lead's decision @@ -357,19 +356,19 @@ func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b } } -func TestAnUpstreamArtifactIsStillRefusedKeyNeverBuiltAgain(t *testing.T) { // 1607/AC2, ADR 0478 clause 2 - r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream"}) +func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4 + r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}) v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, RefusedPrefix) || !strings.Contains(v.Error, "upstream") || !strings.Contains(v.Error, "build-agent") { - t.Fatalf("build = %+v, want failed 'refused kind: upstream' naming build-agent", v) + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotPinned) || !strings.Contains(v.Error, "up") { + t.Fatalf("build = %+v, want failed 'not pinned:' naming the artifact", v) } if len(r.bm.calls) != 0 { - t.Errorf("%d calls, want none for a refused kind", len(r.bm.calls)) + t.Errorf("%d calls, want none for an unpinned upstream", len(r.bm.calls)) } writes := r.st.writes r.consider(t, "modules/m") if r.st.writes != writes { - t.Errorf("a refused key was written again (%d writes, was %d)", r.st.writes, writes) + t.Errorf("an unpinned upstream's key was written again (%d writes, was %d)", r.st.writes, writes) } } @@ -390,14 +389,33 @@ func TestAnUnreadableManifestFailsTheBuild(t *testing.T) { } } -func TestAModuleWithNoFingerprintOrNoArtifactIsNotBuilt(t *testing.T) { // 1607/AC2 +func TestAModuleWithNoFingerprintIsNotBuilt(t *testing.T) { // 1607/AC2 r := newRig(t) - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"}) r.b.Module(context.Background(), "n", data) r.b.Wait() if len(r.st.vals) != 0 { - t.Errorf("keys %v were written: a module with no new-form fingerprint or no artifact has nothing to build", r.st.vals) + t.Errorf("keys %v were written: a module with no new-form fingerprint has nothing to build", r.st.vals) + } +} + +func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testing.T) { // issue 1613, ADR 0478 clause 1 + for name, manifest := range map[string]catalogue.Manifest{"no build": {Module: "m"}, "no artifact": {Module: "m", Build: &catalogue.Build{}}} { + r := newRig(t) + r.mf.byDigest[mdigest('a')] = manifest + v := r.consider(t, "") // a module with nothing to build needs no directory + if v.State != delivery.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" { + t.Errorf("%s: build = %+v, want done with no artifact", name, v) + } + if len(r.bm.calls) != 0 { + t.Errorf("%s: %d build module calls, want none", name, len(r.bm.calls)) + } + writes := r.st.writes + r.b.tried = map[string]bool{} + r.consider(t, "") + if r.st.writes != writes { + t.Errorf("%s: the done key was written again", name) + } } } @@ -421,7 +439,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t * // whose compare-and-set comes first, and the other's write is refused, never overwriting it. func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4 r := newRig(t, goBundle) - other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf} + other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf} value := moduleValue(t, "modules/m") var wg sync.WaitGroup for _, b := range []*Builder{r.b, other} { @@ -432,8 +450,8 @@ func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/A if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 { t.Errorf("build = %+v, want one done key", v) } - if n := len(r.bm.callsOf("build_go_build")); n < 2 || n%2 != 0 { - t.Errorf("%d builds, want two for each instance that built", n) + if n := len(r.bm.callsOf("build_go_build")); n < 1 { + t.Errorf("%d builds, want one for each instance that built", n) } } @@ -570,43 +588,6 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi } } -func TestASecondGoNodeDoesNotBuildOrPushTheImageAgain(t *testing.T) { // review r1, mesh-delivery 2; ADR 0487 clause 8 - r := newRig(t, goBundle, imageArt) - r.b.Health = imageHave(t, "ace", "novox") - r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies for Go (and the image) - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || len(OneNodeModules(v)) != 1 { - t.Fatalf("build = %+v, want done with the Go bundle on one node", v) - } - var image delivery.BuiltArtifact - for _, a := range v.Artifacts { - if a.Kind == "image" { - image = a - } - } - if image.Reference == "" { - t.Fatalf("artifacts = %+v, want a published image", v.Artifacts) - } - r.pres.Put("ace", presenceValue(t, true), false) - r.b.Reconsider(context.Background()) - r.b.Wait() - v = valueOf(t, r.st, r.key) - if len(OneNodeModules(v)) != 0 || len(v.Artifacts) != 2 { - t.Fatalf("build = %+v, want done on two Go nodes", v) - } - if n := len(r.bm.callsOf("build_image_build")); n != 1 { - t.Errorf("%d image builds, want the one of the first build only", n) - } - for _, a := range v.Artifacts { - if a.Kind == "image" && a != image { - t.Errorf("image = %+v, was %+v: the done key was rewritten with another image", a, image) - } - } - if n := len(r.bm.callsOf("build_go_build")); n != 3 { // one, then two - t.Errorf("%d go builds, want 3", n) - } -} - func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5 r := newRig(t, imageArt) r.b.Health = imageHave(t, "ace") diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 28d9afa..3fd061d 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -90,8 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { - t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node)", m.Invokes) + if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 6d36190..460ae22 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -25,6 +25,7 @@ "mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", + "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch" diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 2fef5a6..a5d7c6c 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -7,8 +7,10 @@ import ( "encoding/json" "fmt" "os" + "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-checker/internal/check" ) @@ -23,6 +25,71 @@ func read(s stdio.KeptState) check.StateRead { }} } +// stateStore is a state the tools write: the tries. +type stateStore struct { + check.StateRead + create func(key string, value []byte) error +} + +func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() } +func (s stateStore) Get(key string) (json.RawMessage, error) { return s.StateRead.Get(key) } +func (s stateStore) Create(key string, value []byte) error { return s.create(key, value) } + +// Trying is what the try's tools work on: the tries, and the gate's trunk heads. +type Trying struct { + Tries check.TryStore + Trunks func() (map[string]string, error) + Now func() time.Time +} + +// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them. +func TryTools(tr Trying) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } + return []stdio.Tool{ + {Name: check.TryTool, + Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " + + "its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " + + "where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " + + "the checker checks no level of (hq, one whose trunk the gate does not name) is refused.", + Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/"}, + "head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"}, + "trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}, + "tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"}, + "level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}}, + Run: func(args map[string]any) (any, error) { + if tr.Tries == nil || tr.Trunks == nil { + return nil, fmt.Errorf("the tries state cannot be written here") + } + trunks, err := tr.Trunks() + if err != nil { + return nil, fmt.Errorf("the trunk heads cannot be read: %w", err) + } + now := time.Now + if tr.Now != nil { + now = tr.Now + } + id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"), + Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now()) + if err != nil { + return nil, err + } + return map[string]any{"id": id}, nil + }}, + {Name: check.TryReadTool, + Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " + + "the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{ + "id": map[string]any{"type": "string", "description": "the try's id, try-"}}}, + Run: func(args map[string]any) (any, error) { + if tr.Tries == nil { + return nil, fmt.Errorf("the tries state cannot be read here") + } + return check.ReadTry(tr.Tries, str(args, "id")) + }}, + } +} + // Tools are the read tools over the checker's states. func Tools(r check.Reader) []stdio.Tool { return []stdio.Tool{ @@ -59,12 +126,69 @@ func Tools(r check.Reader) []stdio.Tool { kept, err := r.Kept(level, digest) return map[string]any{"kept": kept}, err }}, + {Name: check.PreviewsReadTool, + Description: "The merge previews the checker holds, each as stored under previews...: the merge commit, per module directory " + + "its tree, manifest digest and lock hashes, or the files in conflict. With a repository, that repository's; with a head (7 or more characters) " + + "only that head's. Keys that are no preview key are listed as unreadable. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/"}, + "head": map[string]any{"type": "string", "description": "a pull request head commit, or a prefix of at least 7 characters"}}}, + Run: func(args map[string]any) (any, error) { + repository, _ := args["repository"].(string) + head, _ := args["head"].(string) + if head != "" && len(head) < 7 { + return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head) + } + ps, bad, err := r.Previews(repository, head) + return map[string]any{"previews": ps, "unreadable": bad}, err + }}, + {Name: check.StatusReadTool, + Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " + + "delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " + + "candidates without a verdict for their heads. The steps the checker holds waiting for a place are in its process, not in a bucket, and the " + + "answer says so. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return r.Status() }}, } } +func capacity() (string, error) { + path := os.Getenv("MESH_CHECKER_SETTINGS") + if path == "" { + return os.Getenv("MESH_CHECKER_CAPACITY"), nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + var s struct { + Capacity string `json:"capacity"` + } + return s.Capacity, json.Unmarshal(raw, &s) +} + func main() { - r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates"))} - if err := stdio.Serve("", Tools(r)); err != nil { + r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")), + PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")), + StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, + Capacity: capacity} + tries := stdio.State(check.TriesState) + source := stdio.State("mesh-delivery-gate.source") + trying := Trying{ + Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error { + _, err := tries.Create(key, json.RawMessage(value)) + return err + }}, + Trunks: func() (map[string]string, error) { + e, err := source.Get(delivery.SourceMainKey) + if err != nil || e == nil { + return nil, err + } + s, err := delivery.DecodeSource(e.Value) + return s.Heads, err + }, + } + if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 5fec12b..4bafd10 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -1,12 +1,17 @@ package main import ( + "encoding/json" + "errors" + "slices" + "strings" "testing" + "time" "mesh-delivery-checker/internal/check" ) -func TestTheToolsServedAreTheThreeReadTools(t *testing.T) { +func TestTheToolsServedAreTheFiveReadTools(t *testing.T) { var names []string for _, tl := range Tools(check.Reader{}) { names = append(names, tl.Name) @@ -14,14 +19,63 @@ func TestTheToolsServedAreTheThreeReadTools(t *testing.T) { t.Errorf("%s has no description or no run", tl.Name) } } - want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool} - if len(names) != 3 || names[0] != want[0] || names[1] != want[1] || names[2] != want[2] { + want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool} + if !slices.Equal(names, want) { t.Errorf("tools = %v, want %v", names, want) } // With nothing to read from a tool says so and changes nothing. for _, tl := range Tools(check.Reader{}) { + if tl.Name == check.StatusReadTool { + continue // it answers from what it can read, and says what it could not + } if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil { t.Errorf("%s answered with no state to read", tl.Name) } } } + +// memStore is a tries state in memory. +type memStore struct{ vals map[string]json.RawMessage } + +func (m *memStore) Keys() ([]string, error) { + var out []string + for k := range m.vals { + out = append(out, k) + } + return out, nil +} +func (m *memStore) Get(key string) (json.RawMessage, error) { return m.vals[key], nil } +func (m *memStore) Create(key string, value []byte) error { + if _, ok := m.vals[key]; ok { + return errors.New("exists") + } + m.vals[key] = value + return nil +} + +func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try + store := &memStore{vals: map[string]json.RawMessage{}} + tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) }, + Trunks: func() (map[string]string, error) { + return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil + }}) + if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" { + t.Fatalf("tools = %+v", tools) + } + out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"}) + if err != nil { + t.Fatal(err) + } + id := out.(map[string]any)["id"].(string) + if id != "try-1000000005" { + t.Errorf("id = %q", id) + } + raw, err := tools[1].Run(map[string]any{"id": id}) + if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) { + t.Errorf("read = %s, %v", raw, err) + } + if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil || + err.Error() != "no level of novox/hq is checked by the checker" { + t.Errorf("hq: %v", err) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go index e3489c3..6415687 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go @@ -57,6 +57,9 @@ func (s kvStore) Keys(ctx context.Context) ([]string, error) { return keys, err } +// Delete removes a key (an ended try past its keeping). +func (s kvStore) Delete(ctx context.Context, key string) error { return s.kv.Delete(ctx, key) } + func opOf(e jetstream.KeyValueEntry) bool { return e.Operation() == jetstream.KeyValuePut } // settle reads the entries a watcher holds now, up to its mark of the last values, and returns the watcher for the diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go index 31702bf..b3aabfa 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go @@ -3,10 +3,13 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "regexp" "strings" + "github.com/nats-io/nats.go" + "mesh-delivery-checker/internal/check" ) @@ -27,6 +30,16 @@ type forgeReads struct { call func(key string, body any) (json.RawMessage, error) } +// ask calls a forge tool; a call that reached nobody (no responders: the forge's tool runner re-serving, or the forge +// down) is marked Unheard, so the check fails at the forge and is judged again when the nodes change (issue 1624). +func (f forgeReads) ask(key string, body any) (json.RawMessage, error) { + raw, err := f.call(key, body) + if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) { + return nil, &check.Unheard{Err: err} + } + return raw, err +} + func ownerRepo(repository string) (string, string) { owner, repo, _ := strings.Cut(repository, "/") return owner, repo @@ -35,7 +48,7 @@ func ownerRepo(repository string) (string, string) { // File is the text of a file of a repository at a ref. func (f forgeReads) File(_ context.Context, repository, path, ref string) (string, error) { owner, repo := ownerRepo(repository) - raw, err := f.call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) + raw, err := f.ask(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) if err != nil { if notFoundStatus.MatchString(err.Error()) { // The forge answers 404 for a file the ref does not hold (issue 1845). The status is matched in its own place, @@ -59,7 +72,7 @@ func (f forgeReads) File(_ context.Context, repository, path, ref string) (strin // not. func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, error) { owner, repo := ownerRepo(repository) - raw, err := f.call(compareTool, map[string]string{"owner": owner, "repo": repo, "base": "main", "head": commit}) + raw, err := f.ask(compareTool, map[string]string{"owner": owner, "repo": repo, "base": "main", "head": commit}) if err != nil { return false, err } @@ -75,7 +88,7 @@ func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, // FullSHA is the whole sha of a commit named by its first digits: the newest commit of the history at that ref. func (f forgeReads) FullSHA(_ context.Context, repository, commit string) (string, error) { owner, repo := ownerRepo(repository) - raw, err := f.call(commitsTool, map[string]any{"owner": owner, "repo": repo, "ref": commit, "limit": 1}) + raw, err := f.ask(commitsTool, map[string]any{"owner": owner, "repo": repo, "ref": commit, "limit": 1}) if err != nil { return "", err } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go index e7ecc85..d6487d6 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/nats-io/nats.go" + "mesh-delivery-checker/internal/check" ) @@ -86,3 +88,18 @@ func TestAFileTheRefDoesNotHoldIsNotFoundAndOtherFailuresAreNot(t *testing.T) { t.Errorf("a 500 is %v, want a failure that is not ErrNotFound", err) } } + +func TestAForgeReadThatReachesNobodyIsUnheard(t *testing.T) { // 1624 C7: "nats: no responders" on mesh-catalog #293 + f := forgeReads{call: func(string, any) (json.RawMessage, error) { return nil, nats.ErrNoResponders }} + _, err := f.File(context.Background(), "novox/mesh-catalog", "checks/agenthome/go.mod", "950e37a46e8d") + var u *check.Unheard + if !errors.As(err, &u) { + t.Errorf("err = %v, want it marked as reaching nobody", err) + } + if _, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "abc"); !errors.As(err, &u) { + t.Errorf("compare: %v", err) + } + if _, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "abc"); !errors.As(err, &u) { + t.Errorf("commits: %v", err) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index cbf005b..e2f4c15 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -19,7 +19,10 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-checker/internal/check" ) // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the @@ -267,7 +270,7 @@ func startBed(t *testing.T) *bed { for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory, delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory, delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory, - delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory} { + delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) if err != nil { t.Fatal(err) @@ -590,3 +593,91 @@ func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T t.Errorf("%d test steps ran after a stale pin", n) } } + +// triesStore is the tries bucket as the tools write it. +type triesStore struct { + kv jetstream.KeyValue + ctx context.Context +} + +func (s triesStore) Keys() ([]string, error) { + keys, err := s.kv.Keys(s.ctx) + if err == jetstream.ErrNoKeysFound { + return nil, nil + } + return keys, err +} +func (s triesStore) Get(key string) (json.RawMessage, error) { + e, err := s.kv.Get(s.ctx, key) + if err == jetstream.ErrKeyNotFound { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} +func (s triesStore) Create(key string, value []byte) error { + _, err := s.kv.Create(s.ctx, key, value) + return err +} + +// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker, +// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no +// verdict and no status owed (ADR 0501 §5). +func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) { + b := startBed(t) + b.start(t, nil) + store := triesStore{b.buckets[check.TriesBucket], b.ctx} + id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now()) + if err != nil { + t.Fatal(err) + } + var last check.Try + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + raw, _ := store.Get(id) + if last, err = check.DecodeTry(raw); err == nil && last.Over() { + break + } + } + if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" { + t.Fatalf("try = %+v, want passed on the trunk", last) + } + if b.fake.count("mesh-build-go.build_go_fetch") == 0 { + t.Errorf("the try fetched nothing") + } + for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} { + if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 { + t.Errorf("a try wrote %s: %v", name, keys) + } + } +} + +// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among +// the readable ones, which is what grants the forge module its read of owed (ADR 0495 §2-§3, ADR 0521; R4e finding R2-1). +func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1 + b := startBed(t) + if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { + t.Fatal(err) + } + b.start(t, nil) + events, err := b.js.Stream(b.ctx, "EVENTS") + if err != nil { + t.Fatal(err) + } + var said announce.States + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + m, err := events.GetLastMsgForSubject(b.ctx, announce.Subject("mesh-delivery-checker")) + if err != nil { + continue + } + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + break + } + if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") || + !said.Readable("previews") || !said.Readable("verdicts") || said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 { + t.Fatalf("the checker said %+v, want its five states with owed, previews and verdicts readable", said) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 6dc5381..ed8ff96 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -3,6 +3,7 @@ package main import ( "context" "sort" + "strings" "sync" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -31,7 +32,7 @@ type judgeDeps struct { func newJudge(d judgeDeps) check.Judge { return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} } // manifestKV reads a manifest by its digest. @@ -47,10 +48,13 @@ type repoModules struct { logf func(format string, args ...any) mu sync.Mutex - langs map[string]map[string]bool // by manifest digest: the languages of its artifacts - repoOf map[string]string // by module: its repository - commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs) - typed map[string]string // by module: its manifest digest + langs map[string]map[string]bool // by manifest digest: the languages of its artifacts + tsPkgs map[string][]string // by manifest digest: the names of its TypeScript package artifacts + path map[string]string // by module: its directory in its repository + named map[string]map[string][]string // by module: each artifact's named paths, as the indexer keeps them + repoOf map[string]string // by module: its repository + commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs) + typed map[string]string // by module: its manifest digest } // Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names. @@ -58,11 +62,14 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted r.mu.Lock() if r.repoOf == nil { r.repoOf, r.typed, r.langs, r.commit = map[string]string{}, map[string]string{}, map[string]map[string]bool{}, map[string]string{} + r.tsPkgs, r.path, r.named = map[string][]string{}, map[string]string{}, map[string]map[string][]string{} } if deleted { delete(r.repoOf, key) delete(r.typed, key) delete(r.commit, key) + delete(r.path, key) + delete(r.named, key) r.mu.Unlock() return } @@ -75,12 +82,15 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted r.repoOf[key] = m.Repository r.typed[key] = m.Manifest r.commit[key] = m.Commit + r.path[key] = m.Path + r.named[key] = m.ArtifactPaths _, known := r.langs[m.Manifest] r.mu.Unlock() if m.Manifest == "" || known { return } set := map[string]bool{} + var pkgs []string if mk, err := delivery.ManifestsKey(m.Manifest); err == nil { if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found { if kept, err := delivery.DecodeManifest(raw); err == nil { @@ -89,6 +99,9 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted if a.Language != "" { set[a.Language] = true } + if a.Kind == "package" && a.Language == "typescript" { + pkgs = append(pkgs, a.Name) + } } } else if err != nil && r.logf != nil { r.logf("the manifest %s cannot be read, so its module counts as no TypeScript: %v", m.Manifest, err) @@ -98,9 +111,37 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted } r.mu.Lock() r.langs[m.Manifest] = set + r.tsPkgs[m.Manifest] = pkgs r.mu.Unlock() } +// PackagePaths are the paths, relative to the repository, that the TypeScript package artifacts of a repository's modules +// are built from, sorted: the named paths the indexer keeps in each module's value (delivery.Module.ArtifactPaths; only the +// indexer reads them from a manifest, the lead's decision L8), under the module's directory. None for a module whose value +// holds no paths. +func (r *repoModules) PackagePaths(repository string) []string { + r.mu.Lock() + defer r.mu.Unlock() + var out []string + for module, repo := range r.repoOf { + if repo != repository { + continue + } + dir := strings.Trim(r.path[module], "/") + for _, name := range r.tsPkgs[r.typed[module]] { + for _, p := range r.named[module][name] { + p = strings.Trim(strings.TrimPrefix(p, "./"), "/") + if dir != "" && dir != delivery.RootPath { + p = dir + "/" + p + } + out = append(out, p) + } + } + } + sort.Strings(out) + return out +} + // BuildModules are the build modules that judge a repository, sorted. func (r *repoModules) BuildModules(repository string) []string { out := []string{buildrun.GoBuildModule} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 28329bf..8afa3c1 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -73,3 +73,40 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test t.Errorf("a repository with no module: %v, want the Go build module", got) } } + +// sdkManifest is mesh-sdk's own manifest (its module.json at feat/r4e-sdk): one TypeScript package naming its paths. +const sdkManifest = `{"module":"mesh-sdk","version":"1","slug":"sdk","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript",` + + `"paths":["src","test","package.json","package-lock.json","tsconfig.json"]}]},"resources":[]}` + +func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *testing.T) { // ADR 0500 §5 with lead decision L8 + store := memManifests{} + data, err := delivery.EncodeManifest(delivery.Manifest{Module: "mesh-sdk", JSON: sdkManifest}) + if err != nil { + t.Fatal(err) + } + store[delivery.ManifestDigest([]byte(sdkManifest))] = data + r := &repoModules{manifests: store} + put := func(path string, paths map[string][]string) { + v, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: strings.Repeat("a", 40), FP: "src1:" + strings.Repeat("a", 64), + Manifest: delivery.ManifestDigest([]byte(sdkManifest)), Path: path, ArtifactPaths: paths}) + if err != nil { + t.Fatal(err) + } + r.Put(context.Background(), "mesh-sdk", v, false) + } + put(".", map[string][]string{"lib": {"src", "package.json"}}) // the indexer's word, not the manifest's five paths + if got := strings.Join(r.PackagePaths("novox/mesh-sdk"), ","); got != "package.json,src" { + t.Errorf("paths = %q, want the indexer's, at the repository's root", got) + } + put("ts", map[string][]string{"lib": {"src"}}) + if got := strings.Join(r.PackagePaths("novox/mesh-sdk"), ","); got != "ts/src" { + t.Errorf("paths = %q, want them under the module's directory", got) + } + put(".", nil) // a value written before the indexer kept paths: nothing to serve by + if got := r.PackagePaths("novox/mesh-sdk"); len(got) != 0 { + t.Errorf("paths = %v, want none", got) + } + if got := r.PackagePaths("novox/mesh-delivery"); len(got) != 0 { + t.Errorf("another repository: %v", got) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go new file mode 100644 index 0000000..e19c92c --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "encoding/json" + "os" + "sort" + "testing" +) + +// The checker says on the bus which states it keeps and which others may read (novox/hq ADR 0495 clauses 2 and 3): a +// reader is granted from that announcement. owed is read by the forge module, which writes mesh/check from it (ADR 0521, +// R4e round 2 finding R2-1), and by the forge bridge; previews by the indexer; verdicts by the gate. The list names every +// state of the manifest, so no state is refused at the call for want of being announced (clause 5). +func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { // R2-1, ADR 0495 §2-§3, ADR 0521 + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + State []struct{ Name string } `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var manifest, announced, readable []string + for _, s := range m.State { + manifest = append(manifest, s.Name) + } + for _, s := range keptStates { + announced = append(announced, s.Name) + if s.Readable { + readable = append(readable, s.Name) + } + } + sort.Strings(manifest) + sort.Strings(announced) + if !equal(manifest, announced) { + t.Errorf("announced %v, the manifest keeps %v", announced, manifest) + } + if want := []string{"owed", "previews", "verdicts"}; !equal(readable, want) { + t.Errorf("readable %v, want %v", readable, want) + } +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 40ed0d7..7026c9f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -25,6 +25,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" @@ -113,6 +114,16 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti } // run is the program: it returns when ctx ends or the bus cannot be used. +// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module, +// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate. +var keptStates = []announce.State{ + {Name: "kept"}, + {Name: "owed", Readable: true}, + {Name: "previews", Readable: true}, + {Name: "tries"}, + {Name: "verdicts", Readable: true}, +} + func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { logger := log.New(logOut, "mesh-delivery-checker: ", log.LstdFlags) path := strings.TrimSpace(getenv("MESH_BROKER_FILE")) @@ -151,7 +162,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } buckets := map[string]jetstream.KeyValue{} for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, - delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket} { + delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} { if buckets[name], err = bucket(name); err != nil { return err } @@ -167,6 +178,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer caller.Close() + // What the checker keeps, said on the bus (ADR 0495 §2-§3): a reader is granted from it. Not fatal when it cannot be said + // now: process.Conn says it again at every reconnect. + if err := caller.Keeps(keptStates...); err != nil { + logger.Printf("what the checker keeps could not be said on the bus, so readers of owed, previews and verdicts are not granted until it is: %v", err) + } stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", defaultStepBound) if err != nil { @@ -209,8 +225,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro default: builds = buildReader{kvStore{kv}} } - holds := newHoldWatch(ctx, js, ch.Reconsider, logger) + // The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries. + tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf} + holds := newHoldWatch(ctx, js, func(ctx context.Context) { ch.Reconsider(ctx); tryer.Reconsider(ctx) }, logger) ch.Hold = holds.Held + tryer.Hold = holds.Held announce := func(ctx context.Context, event, key string) { body, _ := json.Marshal(map[string]string{"key": key}) if _, err := js.Publish(ctx, eventSubject(event), body); err != nil { @@ -225,6 +244,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro announce: announce, }) ch.Announce = announce + tryer.Judge = ch.Judge // The toolchains first, so a check that starts from the first candidate finds the nodes. watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger) @@ -308,7 +328,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { - ch.Reconsider(ctx) // the trunk moved: a verdict for the old trunk is judged again + ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again } }) }() @@ -336,11 +356,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ch.Sweep(ctx); err != nil { return err } + // The tries: a queued one starts, one left running is taken over, an ended one past its keeping is removed. + tw, err := settle(ctx, buckets[check.TriesBucket], func(jetstream.KeyValueEntry) {}) + if err != nil { + return err + } + tryer.Reconsider(ctx) + go func() { + errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) { + if opOf(e) { + tryer.Put(ctx, e.Key(), e.Value()) + } + }) + }() logger.Printf("started on %s", cred.Node) go func() { errs <- follow(ctx, w, each) }() select { case <-ctx.Done(): ch.Wait() + tryer.Wait() return nil case err := <-errs: return err diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 9c911a8..0de4a55 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 3515fb8..db1ea18 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 24cb19e..4240ed4 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -4,6 +4,7 @@ import ( "fmt" "maps" "sort" + "strings" "sync" "time" @@ -35,13 +36,17 @@ type Case struct { Key string Group string HeadsHash string - Pulls []Pull + // Heads is the heads hash the gate matches an Approve's members by (GateHeads): a verdict carries it. + Heads string + Pulls []Pull // Trunk is the trunk head of each repository of Pulls. Trunk map[string]string // Left says why pull requests of the candidate are not judged (hq's, a repository with no known trunk); empty when all are. Left []string // Queued is when the checker took the case up, for the time of its queue. Queued time.Time + // Try narrows a try's check (its level, its tests); nil for a pull request's. + Try *TryOptions // Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at. Progress *Progress } @@ -75,7 +80,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case { if err != nil { return nil } - cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Trunk: map[string]string{}, Progress: &Progress{}} + cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}} for _, p := range c.Pulls { switch head, known := trunk[p.Repository]; { case p.Repository == HQRepository: @@ -99,6 +104,22 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case { return cs } +// GateHeads is the hash the gate matches the members an Approve covers by (mesh-catalog mesh-delivery-gate onestatus.go +// headsOf): delivery.HeadsHash over every pull request of the candidate, hq's included, as +// `#=`. A lone pull request is a group of one. Empty for a candidate +// with no pull request. +func GateHeads(c delivery.Candidate) string { + heads := make([]delivery.Head, 0, len(c.Pulls)) + for _, p := range c.Pulls { + heads = append(heads, delivery.Head{Repository: strings.ToLower(p.Repository), Number: p.Number, Head: strings.ToLower(p.Head.Head)}) + } + hash, err := delivery.HeadsHash(heads) + if err != nil { + return "" + } + return hash +} + // sameTrunk says whether two verdicts' trunk heads are the same. func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) } diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go index 84753ed..1236af4 100644 --- a/modules/mesh-delivery-checker/internal/check/checker.go +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -18,6 +18,10 @@ const DefaultBound = 30 * time.Minute // is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8). const NodesWords = "build node qualifies" +// UnheardWords end the Words of a failure because a part of the mesh the check reads was not heard (the forge, issue 1624): +// like a failure of the nodes, it is not the change's, and is judged again when the nodes change. +const UnheardWords = "not heard; judged again when the nodes change" + // The events the checker says on its stream checks (module.json): a preview or a status was written, so a part that reads // the bucket, which is made when the checker is assigned and may not exist when that part starts, finds it (ADR 0437 §8: no // timer). The body is {"key": ...}. @@ -234,7 +238,7 @@ type claimed struct{ rev uint64 } // - a pass or a failure for the trunk heads the case has stands; one for other trunk heads (the trunk moved) is taken // again by compare-and-set on the same key, and so is a failure of the nodes after the nodes changed. func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64) (*claimed, error) { - running := delivery.Verdict{Group: cs.Group, Heads: cs.HeadsHash, Trunk: cs.Trunk, State: delivery.VerdictRunning} + running := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictRunning} data, err := delivery.EncodeVerdict(running) if err != nil { return nil, err @@ -287,7 +291,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64 // IsNodesFailure says whether a verdict failed because no build node could take the check. func IsNodesFailure(v delivery.Verdict) bool { - return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, NodesWords) + return v.State == delivery.VerdictFail && (strings.Contains(v.Failed.Words, NodesWords) || strings.Contains(v.Failed.Words, UnheardWords)) } // try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends @@ -313,14 +317,14 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { return } v := out.Verdict - v.Group, v.Heads, v.Trunk = cs.Group, cs.HeadsHash, cs.Trunk + v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) { v.Took.Queue = began.Sub(cs.Queued).Milliseconds() } data, err := delivery.EncodeVerdict(v) if err != nil { ch.logf("%s: the verdict cannot be written: %v", cs.Key, err) - v = delivery.Verdict{Group: cs.Group, Heads: cs.HeadsHash, Trunk: cs.Trunk, State: delivery.VerdictFail, + v = delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "checker", Words: "the verdict could not be written: " + err.Error()}} if data, err = delivery.EncodeVerdict(v); err != nil { return @@ -335,6 +339,13 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { ch.nodesFail[cs.Key] = epoch ch.mu.Unlock() } + if ch.movedSince(cs) { + // Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the + // check of the new trunk, which Reconsider starts, ends. + ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil) + ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State) + return + } ch.writeOwed(ctx, cs, v, out.Builds, out.Slowest) ch.logf("%s: %s", cs.Key, v.State) } @@ -342,25 +353,7 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { // judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of // failure naming the machine and the step. func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome { - bound := ch.bound() - run, cancel := context.WithTimeoutCause(ctx, bound, errBound) - defer cancel() - out, err := ch.Judge.Judge(run, cs) - stage, node := cs.Progress.Now() - if node == "" { - node = "checker" - } - switch { - case ctx.Err() != nil: - return Outcome{} - case context.Cause(run) == errBound: - if stage == "" { - stage = "starting" - } - return Outcome{Verdict: failure(node, fmt.Sprintf("the check ran past its bound of %s, stuck at: %s", bound, stage))} - case err != nil: - return Outcome{Verdict: failure(node, err.Error())} - } + out, _ := judgeWithin(ctx, ch.Judge, cs, ch.bound()) return out } @@ -467,3 +460,44 @@ func (ch *Checker) Sweep(ctx context.Context) error { } return nil } + +// TrunkMoved is Reconsider after a move of a trunk, which first owes mesh/check pending on every open pull request of a +// candidate whose verdict was judged on other trunk heads, every member's for a group (the lead's decision L4, the checker's +// part of issue 1302): the forge's success of the old trunk never stands while the check waits for a place or for a delivery +// that holds it, so the gate never reads it as the head's. +func (ch *Checker) TrunkMoved(ctx context.Context) { + ch.init() + ch.mu.Lock() + known := make(map[string]delivery.Candidate, len(ch.known)) + for k, c := range ch.known { + known[k] = c + } + ch.mu.Unlock() + heads := ch.heads() + for _, c := range known { + cs := CaseOf(c, heads) + if cs == nil { + continue + } + value, _, found, err := ch.Verdicts.Get(ctx, cs.Key) + if err != nil || !found { + continue // never judged: no status to put out of date + } + if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) { + continue + } + ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil) + } + ch.Reconsider(ctx) +} + +// movedSince says whether a trunk the case was judged on is no longer the trunk's head. +func (ch *Checker) movedSince(cs *Case) bool { + now := ch.heads() + for repo, head := range cs.Trunk { + if now[repo] != head { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go index f57ed71..295a9ff 100644 --- a/modules/mesh-delivery-checker/internal/check/checker_test.go +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -19,6 +19,7 @@ const ( trunkA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" trunkA2 = "abababababababababababababababababababab" trunkS = "5555555555555555555555555555555555555555" + trunkS2 = "5656565656565656565656565656565656565656" ) // candidateOf is a candidate of a group with the given pull requests, as the indexer writes one. diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 66edd34..1864f55 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -2,10 +2,14 @@ package check import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" "path/filepath" + "reflect" + "sort" "strings" "sync" "testing" @@ -565,7 +569,7 @@ func trunksOf(t *testing.T, heads map[string]string) *Trunks { return tr } -func TestNothingIsKeptForAWorkspaceGivenSiblingsBeside(t *testing.T) { // review D1, ADR 0444, issue 1287 +func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversThem(t *testing.T) { // review D1, ADR 0444, issue 1287 b := newBench(t) b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA}) b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) @@ -573,7 +577,7 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsBeside(t *testing.T) { // review t.Fatalf("the fetch was not given siblings: %+v", b.steps.tool("fetch")) } if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 { - t.Errorf("kept %v for a package whose tests may read a sibling the digest does not cover", keys) + t.Errorf("kept %v for a package whose digest does not say it covers the siblings", keys) } // And a kept verdict of the same digest from elsewhere is not taken for it: everything runs. if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digA, keptValue("fixture/slow", 2)); err != nil { @@ -585,3 +589,137 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsBeside(t *testing.T) { // review t.Errorf("%d test steps, want both levels run whole", n) } } + +// besideDigest stands for the Go build module's digest over a workspace with siblings (DigestVersion 3, lead decision L5): +// each package's digest folds the siblings' commits in, and the answer echoes the lines it folded. +func besideDigest(b *bench) func(c stepCall) (any, error) { + return func(c stepCall) (any, error) { + level := c.args["level"].(string) + var lines []string + for _, f := range b.steps.tool("fetch") { + refs, _ := f.args["beside"].([]BesideRef) + for _, r := range refs { + lines = append(lines, r.Repository+" "+r.Commit) + } + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(level + "\n" + strings.Join(lines, "\n"))) + return Digested{Level: level, Packages: map[string]string{"fixture/slow": hex.EncodeToString(sum[:])}, Beside: lines}, nil + } +} + +func TestACoreRepositorysPassingPackageIsKeptUnderADigestCoveringItsSiblings(t *testing.T) { // 1287 AC1-AC3, AC7 with L5 + b := newBench(t) + b.steps.handlers["build_go_digest"] = besideDigest(b) + b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA}) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + keys, _ := b.kept.Keys(context.Background()) + if len(keys) != 2 { + t.Fatalf("kept %v, want the passing package at both levels", keys) + } + // The same siblings again: everything is taken, nothing runs. + b.steps.calls = nil + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if n := len(b.steps.tool("test")); n != 0 || v.Levels[delivery.LevelUnit].Run != 0 || v.Levels[delivery.LevelUnit].Kept == 0 { + t.Fatalf("%d test steps, unit %+v, want none run and the kept verdict taken", n, v.Levels[delivery.LevelUnit]) + } + // A sibling's commit moves: a new digest, so the package runs again and is kept under it. + b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS2, "novox/mesh-catalog": trunkA}) + b.steps.calls = nil + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if n := len(b.steps.tool("test")); n != 2 { + t.Errorf("%d test steps after a sibling moved, want both levels run", n) + } + if keys, _ := b.kept.Keys(context.Background()); len(keys) != 4 { + t.Errorf("kept %v, want a second key per level under the new digest", keys) + } +} + +func TestADigestThatEchoesOtherSiblingsKeepsNothing(t *testing.T) { // 1287 with L5: an echo that differs is not trusted + b := newBench(t) + inner := besideDigest(b) + b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) { + v, err := inner(c) + d := v.(Digested) + d.Beside = d.Beside[:len(d.Beside)-1] + return d, err + } + b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA}) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 { + t.Errorf("kept %v under a digest that does not cover every sibling", keys) + } +} + +type pkgPaths map[string][]string + +func (p pkgPaths) PackagePaths(repository string) []string { return p[repository] } + +// groupPackageBench is a group of a TypeScript member (mesh-delivery) and the mesh-sdk member, whose merge changes sdkChanged. +func groupPackageBench(t *testing.T, sdkChanged []string) *bench { + b := newBench(t) + b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}, repoSDK: {buildrun.GoBuildModule, buildrun.NodeBuildModule}} + b.ex.Packages = pkgPaths{repoSDK: {"src", "test", "package.json", "package-lock.json", "tsconfig.json"}} + fetch := func(c stepCall) (any, error) { + f := b.fetch + if c.args["repository"] == repoSDK { + f.Changed = sdkChanged + } + return f, nil + } + b.steps.handlers["build_go_fetch"], b.steps.handlers["build_node_fetch"] = fetch, fetch + return b +} + +func nodeFetchOf(b *bench, repo string) stepCall { + for _, c := range b.steps.tool("fetch") { + if c.module == buildrun.NodeBuildModule && c.args["repository"] == repo { + return c + } + } + return stepCall{} +} + +func TestATypeScriptMemberTakesTheGroupsPackageWhenTheSDKMemberChangesItsPaths(t *testing.T) { // ADR 0500 §5, C11b + b := groupPackageBench(t, []string{"go/delivery/x.go", "src/state.ts"}) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))) + want := map[string]any{"repository": repoSDK, "commit": head2, "ref": "refs/pull/12/head"} + if got := nodeFetchOf(b, repoA).args["package"]; !reflect.DeepEqual(got, want) { + t.Errorf("mesh-delivery's Node fetch was given the package %v, want %v", got, want) + } + if got := nodeFetchOf(b, repoSDK).args["package"]; got != nil { + t.Errorf("the mesh-sdk member's own fetch was given %v: it builds its package itself", got) + } +} + +func TestATypeScriptMemberResolvesTheRegistryWhenTheSDKMemberLeavesItsPaths(t *testing.T) { // ADR 0500 §5: the manifest alone, or Go alone + for _, changed := range [][]string{{"go/delivery/x.go"}, {"module.json"}, {"srcx/a.ts"}} { + b := groupPackageBench(t, changed) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))) + if got := nodeFetchOf(b, repoA).args["package"]; got != nil { + t.Errorf("%v: the Node fetch was given %v, want the registry", changed, got) + } + } + // A lone TypeScript pull request: no group package either. + b := groupPackageBench(t, []string{"src/state.ts"}) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if got := nodeFetchOf(b, repoA).args["package"]; got != nil { + t.Errorf("a lone pull request's Node fetch was given %v", got) + } +} + +func TestNoStepOfACheckCarriesPublish(t *testing.T) { // ADR 0500 §4, 1309 (the checker's half): only the builder's trunk build publishes + b := groupPackageBench(t, []string{"src/state.ts"}) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))) + if len(b.steps.calls) == 0 { + t.Fatal("no step ran") + } + for _, c := range b.steps.calls { + if _, ok := c.args["publish"]; ok { + t.Errorf("%s was given publish: %v", c.tool, c.args) + } + if strings.HasSuffix(c.tool, "_build") || strings.HasSuffix(c.tool, "_pack") { + t.Errorf("the checker asked %s: a check builds no artifact of its own", c.tool) + } + } +} diff --git a/modules/mesh-delivery-checker/internal/check/gate_test.go b/modules/mesh-delivery-checker/internal/check/gate_test.go new file mode 100644 index 0000000..6c709e0 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/gate_test.go @@ -0,0 +1,106 @@ +package check + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// What the gate reads of the checker (R4e, lead's interface with dev-r4e-gate): the verdict's heads are the gate's hash over +// `#=`, a lone pull request is a group of one, and a move of a trunk marks +// mesh/check pending on every open pull request into it, every member's for a group, before any check starts. + +func owedState(t testing.TB, r *rig, repo string, number int, head string) string { + t.Helper() + key, err := delivery.CheckerOwedKey(repo, number, head) + if err != nil { + t.Fatal(err) + } + raw, _, found, _ := r.owed.Get(context.Background(), key) + if !found { + return "" + } + o, err := delivery.DecodeCheckerOwed(raw) + if err != nil { + t.Fatal(err) + } + return o.State +} + +func TestTheVerdictsHeadsAreTheGatesHashOverLowercaseHeads(t *testing.T) { // gate (b), (c) + ctx := context.Background() + r := newRig(t, passJudge()) + upper := strings.ToUpper(head1[:20]) + head1[20:] + r.setTrunks(t, map[string]string{"Novox/Mesh-Delivery": trunkA, repoA: trunkA, repoSDK: trunkS}) + cand := candidateOf(t, "feat/x", pullOf("Novox/Mesh-Delivery", 7, upper)) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + want, _ := delivery.HeadsHash([]delivery.Head{{Repository: repoA, Number: 7, Head: head1}}) + if v := r.verdict(t, keyOf(t, "feat/x", cand)); v.Heads != want { + t.Errorf("heads = %q, want the gate's hash of the lone pull request, a group of one, %q", v.Heads, want) + } +} + +func TestAGroupsVerdictHeadsCoverEveryMember(t *testing.T) { // gate (b): hq's member too, as the gate's ask covers it + ctx := context.Background() + r := newRig(t, passJudge()) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(HQRepository, 3, head2)) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + want, _ := delivery.HeadsHash([]delivery.Head{{Repository: repoA, Number: 7, Head: head1}, {Repository: repoSDK, Number: 12, Head: head2}, + {Repository: HQRepository, Number: 3, Head: head2}}) + if v := r.verdict(t, keyOf(t, "feat/x", cand)); v.Heads != want { + t.Errorf("heads = %q, want %q", v.Heads, want) + } +} + +func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) { // gate (a), lead decision L4 (1302's checker part) + ctx := context.Background() + r := newRig(t, passJudge()) + r.ch.Candidate(ctx, "feat%2Fg", candidateOf(t, "feat/g", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))) + r.ch.Candidate(ctx, "feat%2Fs", candidateOf(t, "feat/s", pullOf(repoSDK, 13, head1))) + r.ch.Wait() + for _, p := range []struct { + repo string + n int + head string + }{{repoA, 7, head1}, {repoSDK, 12, head2}, {repoSDK, 13, head1}} { + if s := owedState(t, r, p.repo, p.n, p.head); s != "success" { + t.Fatalf("%s#%d is %q before the move", p.repo, p.n, s) + } + } + // A delivery builds, so no check starts; mesh-delivery's trunk moves. + r.ch.Hold = func() bool { return true } + r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) + r.ch.TrunkMoved(ctx) + r.ch.Wait() + if s := owedState(t, r, repoA, 7, head1); s != "pending" { + t.Errorf("mesh-delivery#7, into the moved trunk, is %q, want pending", s) + } + if s := owedState(t, r, repoSDK, 12, head2); s != "pending" { + t.Errorf("mesh-sdk#12, a member of the same group, is %q, want pending", s) + } + if s := owedState(t, r, repoSDK, 13, head1); s != "success" { + t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s) + } +} + +func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown + ctx := context.Background() + var r *rig + j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) { + if call == 1 { + r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs + r.ch.Hold = func() bool { return true } // and a delivery holds the next check + } + return Outcome{Verdict: passing()}, nil + }} + r = newRig(t, j) + r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) + r.ch.Wait() + if s := owedState(t, r, repoA, 7, head1); s != "pending" { + t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index df003a4..5593b58 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "path" + "slices" "sort" "strings" "time" @@ -68,6 +69,11 @@ type Examiner struct { Builds BuildStates // Announce says a preview was written under the key (EventPreviewWritten); nil says nothing. Announce Announcer + // Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from + // the indexer's modules (delivery.Module.ArtifactPaths, lead decision L8); nil serves no group package. + Packages interface { + PackagePaths(repository string) []string + } // Trunks are the trunk heads of every repository, for the siblings a repository's tests read. Trunks interface{ Heads() map[string]string } Now func() time.Time @@ -87,6 +93,16 @@ func (j *Examiner) logf(format string, args ...any) { } } +// besideLines are the siblings as the digest step echoes them: ` `, sorted. +func besideLines(refs []BesideRef) []string { + out := make([]string, 0, len(refs)) + for _, r := range refs { + out = append(out, r.Repository+" "+r.Commit) + } + sort.Strings(out) + return out +} + // target is a pull request on one build module: the node chosen, the workspace its fetch made and what it said. type target struct { pull Pull @@ -98,8 +114,9 @@ type target struct { fetchArgs map[string]any // priority are the Go packages (import paths) or the Node module directories whose test files the merge changes. changed []string - // beside says the workspace was given siblings: its tests may read what the digest does not cover, so nothing is kept for it. - beside bool + // beside are the lines ` ` of the siblings the workspace was given, sorted: its tests may read them, + // so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5). + beside []string plans map[string]*goPlan // by level } @@ -142,6 +159,9 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { return st.finish(ctx, *v), nil } for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { + if c.Try != nil && c.Try.Level != "" && c.Try.Level != level { + continue // a try of one level runs that level alone + } for _, priority := range []bool{true, false} { for _, t := range st.targets { if v := st.runTarget(ctx, t, level, priority); v != nil { @@ -254,7 +274,20 @@ func (st *state) choose(bm string) (node, toolchain string, err error) { func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { began := st.j.now() defer func() { st.clone = st.j.now().Sub(began) }() - for _, p := range st.c.Pulls { + // The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members' + // Node fetches take the group's package (ADR 0500 §5). + pulls := slices.Clone(st.c.Pulls) + slices.SortStableFunc(pulls, func(a, b Pull) int { + if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) { + if a.Repository == SDKRepository { + return -1 + } + return 1 + } + return 0 + }) + var groupPackage map[string]any + for _, p := range pulls { var merged []Fetched for _, bm := range st.j.Modules.BuildModules(p.Repository) { node, tc, err := st.choose(bm) @@ -265,14 +298,17 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { st.nodes[node] = true args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk} t.fetchArgs = args + if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository { + args["package"] = groupPackage + } if bm == buildrun.GoBuildModule { beside, err := st.j.besideFor(ctx, st.c, p) if err != nil { - return fail(node, err.Error()) + return fail(forgeMachine(err, node), err.Error()+unheardWords(err)) } if len(beside) > 0 { args["beside"] = beside - t.beside = true + t.beside = besideLines(beside) } } raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args) @@ -284,18 +320,45 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw)) } if len(t.fetched.Conflicts) > 0 { - st.j.writePreview(ctx, p, append(merged, t.fetched)) + if st.c.Try == nil { + st.j.writePreview(ctx, p, append(merged, t.fetched)) + } return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", "))) } t.changed = t.fetched.Changed merged = append(merged, t.fetched) st.targets = append(st.targets, t) } - st.j.writePreview(ctx, p, merged) + if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head + st.j.writePreview(ctx, p, merged) + } + if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) { + groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)} + } } return nil } +// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's +// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5). +func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool { + if j.Packages == nil { + return false + } + paths := j.Packages.PackagePaths(p.Repository) + for _, f := range merged { + for _, changed := range f.Changed { + for _, named := range paths { + named = strings.TrimSuffix(strings.TrimPrefix(named, "./"), "/") + if changed == named || strings.HasPrefix(changed, named+"/") { + return true + } + } + } + } + return false +} + // isTestFile says whether a repository-relative path is a test file of the languages the checker judges: Go's _test.go and // the TypeScript module's .test. and .spec. files and test directories. func isTestFile(p string) bool { @@ -398,9 +461,8 @@ func (st *state) planGo(ctx context.Context, t *target, level string) *delivery. } plan := &goPlan{digests: map[string]string{}} t.plans[level] = plan - if t.beside { - // The digest covers the package's own files, the toolchain and the header, not the siblings placed beside the tree: a - // package whose tests read one must never be kept, nor taken from a verdict kept for other inputs (ADR 0444). + if st.c.Try != nil && st.c.Try.Run != "" { + // A try of some tests runs them alone: a partial run proves no package, so nothing is taken or kept. plan.all = true return nil } @@ -421,6 +483,13 @@ func (st *state) planGo(ctx context.Context, t *target, level string) *delivery. plan.all = true return nil } + if !slices.Equal(d.Beside, t.beside) { + // The siblings the tests read are not all in the digest (a build module before DigestVersion 3, or another set): a + // package whose tests read one is never kept, nor taken from a verdict kept for other inputs (ADR 0444, L5). + st.j.logf("%s#%d: the %s level's digest covers the siblings %v, not %v, so it runs whole and keeps nothing", t.pull.Repository, t.pull.Number, level, d.Beside, t.beside) + plan.all = true + return nil + } plan.digests = d.Packages a := st.levels[level] pkgs := make([]string, 0, len(d.Packages)) @@ -467,6 +536,9 @@ func (st *state) runGo(ctx context.Context, t *target, level string, priority bo if run != nil { args["packages"] = run } + if st.c.Try != nil && st.c.Try.Run != "" { + args["run"] = st.c.Try.Run // build_go_test's go test -run pattern + } raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), args) if err != nil { return fail(t.node, fmt.Sprintf("the %s level of %s#%d: %v", level, t.pull.Repository, t.pull.Number, err)) diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go index 0882cf1..cedef15 100644 --- a/modules/mesh-delivery-checker/internal/check/line.go +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -2,6 +2,7 @@ package check import ( "fmt" + "slices" "strings" "unicode/utf8" @@ -50,7 +51,7 @@ func firstFailed(v delivery.Verdict) string { // pieces are the parts of a line, from the most to the least needed. type pieces struct { - head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, group string + head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string } func (p pieces) join(drop int) string { @@ -70,7 +71,10 @@ func (p pieces) join(drop int) string { if drop < 2 && p.group != "" { parts = append(parts, p.group) } - if p.composition != "" { + switch { + case drop >= 3 && p.shortComposition != "": + parts = append(parts, p.shortComposition) + case p.composition != "": parts = append(parts, p.composition) } out := make([]string, 0, len(parts)) @@ -90,14 +94,37 @@ func levelWords(name string, r delivery.LevelResult) string { return s } -func compositionWords(c *delivery.Composition) string { - switch { - case c == nil, c.State == delivery.CompositionNotRun: +// compositionWords says the composition: not yet run, or once the assigner's check reports it (issue 1297) passed or +// failed, with the time it took and the machines it composed (issue 1300). +func compositionWords(c *delivery.Composition, took int64) string { return composition(c, took, false) } + +// composition is compositionWords; short counts the machines and leaves the SDK commit to the report. +func composition(c *delivery.Composition, took int64, short bool) string { + if c == nil || c.State == delivery.CompositionNotRun { return "composition not yet run" - case c.State == delivery.CompositionPass: - return "composition passed with " + short12(c.SDK) } - return "composition failed: " + c.Words + in := "" + if took > 0 { + in = " in " + seconds(took) + } + on := "" + if len(c.Machines) > 0 { + machines := slices.Sorted(slices.Values(c.Machines)) + on = " on " + strings.Join(machines, ", ") + switch { + case len(machines) > 1 && short: + on = fmt.Sprintf(" on %d machines", len(machines)) + case len(machines) > 1: + on = fmt.Sprintf(" on %d machines: %s", len(machines), strings.Join(machines, ", ")) + } + } + switch { + case c.State == delivery.CompositionPass && short: + return "composition passed" + in + on // the SDK it composed with is in the report + case c.State == delivery.CompositionPass: + return "composition passed" + in + " with " + short12(c.SDK) + on + } + return "composition failed" + in + on + ": " + c.Words } func buildWords(builds []BuildNote) string { @@ -149,7 +176,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s for _, x := range []struct { name string ms int64 - }{{"queue", t.Queue}, {"clone", t.Clone}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}, {"composition", t.Composition}} { + }{{"queue", t.Queue}, {"clone", t.Clone}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}} { if x.ms > 0 { times = append(times, x.name+" "+seconds(x.ms)) } @@ -181,7 +208,8 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s } pc.builds = buildWords(builds) if v.State != delivery.VerdictRunning { - pc.composition = compositionWords(v.Composition) + pc.composition = compositionWords(v.Composition, v.Took.Composition) + pc.shortComposition = composition(v.Composition, v.Took.Composition, true) } if cs != nil && len(cs.Pulls) > 1 { pc.group = fmt.Sprintf("group %s of %d pull requests", cs.Group, len(cs.Pulls)) diff --git a/modules/mesh-delivery-checker/internal/check/line_test.go b/modules/mesh-delivery-checker/internal/check/line_test.go index 6ea2499..662db1d 100644 --- a/modules/mesh-delivery-checker/internal/check/line_test.go +++ b/modules/mesh-delivery-checker/internal/check/line_test.go @@ -133,3 +133,24 @@ func TestThePassedCompositionIsSaidOnlyWhenItWasRun(t *testing.T) { // ADR 0484 t.Errorf("a verdict with no composition: %q", r) } } + +func TestTheCompositionsTimeAndMachinesAreSaidOnceTheAssignerReportsThem(t *testing.T) { // 1300 (C10), the assigner's verdict (issue 1297) + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + v := passVerdict() + v.Took.Composition = 2400 + v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab", Machines: []string{"g14", "ace", "novox", "shanks"}} + r := Report(v, cs, p, nil, nil) + for _, want := range []string{"composition passed in 2.4s with afbe371003a5 on 4 machines: ace, g14, novox, shanks"} { + if !strings.Contains(r, want) { + t.Errorf("report %q lacks %q", r, want) + } + } + if l := Line(v, cs, p, nil, nil); !strings.Contains(l, "composition passed in 2.4s") || !strings.Contains(l, "on 4 machines") || utf8.RuneCountInString(l) > MaxLine { + t.Errorf("line = %q", l) + } + v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: "novox", Words: "composition refused"} + v.Composition = &delivery.Composition{State: delivery.CompositionFail, Words: "novox: two seats claim dns", Machines: []string{"novox"}} + if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition failed in 2.4s on novox: novox: two seats claim dns") { + t.Errorf("failed composition report = %q", r) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/pins.go b/modules/mesh-delivery-checker/internal/check/pins.go index 326d606..f1e2937 100644 --- a/modules/mesh-delivery-checker/internal/check/pins.go +++ b/modules/mesh-delivery-checker/internal/check/pins.go @@ -77,7 +77,7 @@ func (st *state) checkPins(ctx context.Context) verdictPtr { text, err = st.j.Forge.File(ctx, t.pull.Repository, file, t.pull.Trunk) } if err != nil { - return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: %s cannot be read to check its mesh-sdk pin: %v", t.pull.Repository, t.pull.Number, file, err)) + return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: %s cannot be read to check its mesh-sdk pin: %v%s", t.pull.Repository, t.pull.Number, file, err, unheardWords(err))) } pin, ok := SDKPin(text) if !ok { @@ -88,7 +88,7 @@ func (st *state) checkPins(ctx context.Context) verdictPtr { } onTrunk, err := st.j.Forge.OnTrunk(ctx, SDKRepository, pin) if err != nil { - return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: whether mesh-sdk %s is on its trunk cannot be told: %v", t.pull.Repository, t.pull.Number, pin, err)) + return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: whether mesh-sdk %s is on its trunk cannot be told: %v%s", t.pull.Repository, t.pull.Number, pin, err, unheardWords(err))) } if onTrunk { continue @@ -115,6 +115,14 @@ func pinHint(c *Case, fetchError string) string { return " (a pin to the head of the group's mesh-sdk pull request names a commit only that head carries, which go mod download cannot resolve; " + Issue1148 + ")" } +// unheardWords ends the words of a failed read of the forge that reached nobody: the forge was not heard (UnheardWords). +func unheardWords(err error) string { + if isUnheard(err) { + return " (the forge was " + UnheardWords + ")" + } + return "" +} + // forgeMachine is the machine a failed read of the forge is put at: the forge when nobody answered it (no build node is to // blame, issue 1820), the build node whose workspace was read otherwise. func forgeMachine(err error, node string) string { diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go index 9198b17..703a11b 100644 --- a/modules/mesh-delivery-checker/internal/check/pins_test.go +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -253,3 +253,15 @@ func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // r t.Errorf("verdict = %+v", v) } } + +func TestAForgeThatIsNotHeardFailsAtTheForgeAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // 1624 C7 + b := newBench(t) + b.ex.Forge = &fakeForge{err: &Unheard{Err: fmt.Errorf("nats: no responders available for request")}} + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if v.State != delivery.VerdictFail || v.Failed.Machine != "forge" { + t.Fatalf("verdict = %+v, want failed at the forge", v) + } + if !IsNodesFailure(v) { + t.Errorf("a forge that was not heard is not the change's failure: %+v is not judged again when the nodes change", v.Failed) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/preview.go b/modules/mesh-delivery-checker/internal/check/preview.go index 5d4cf56..bc37e03 100644 --- a/modules/mesh-delivery-checker/internal/check/preview.go +++ b/modules/mesh-delivery-checker/internal/check/preview.go @@ -14,7 +14,7 @@ var digest64 = regexp.MustCompile(`^[0-9a-f]{64}$`) // PreviewOf is what the fetch steps said of a pull request's head merged onto its trunk head, as the value of a previews // key (ADR 0484 clause 3): the merge commit and, for each module directory the fetches named with a manifest, its tree, the -// digest of its manifest and the hashes of its lock files. The fetches of the repository's build modules merge the same +// digest of its manifest, the hashes of its lock files and its named paths' objects (ADR 0499 clause 2). The fetches of the repository's build modules merge the same // trees; the first one's merge commit is kept. A module directory with no tree or no manifest digest is left out, so the // indexer gives its module no fingerprint and says why. A head that does not merge has a preview of the files in conflict // alone, so the indexer says the candidate's modules have no fingerprint because it does not merge (issue 1445). It @@ -43,7 +43,7 @@ func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) { if t.Tree == "" || !digest64.MatchString(t.Manifest) { continue } - mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks} + mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks, Paths: t.Paths} } } if merge == "" { diff --git a/modules/mesh-delivery-checker/internal/check/preview_named_test.go b/modules/mesh-delivery-checker/internal/check/preview_named_test.go new file mode 100644 index 0000000..fccc274 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/preview_named_test.go @@ -0,0 +1,23 @@ +package check + +import ( + "encoding/json" + "reflect" + "testing" +) + +// I8, ADR 0499 clause 2: the preview carries each named path's object a fetch's module trees answer (mesh-sdk +// buildstep.ModuleTree's paths), so the indexer computes the candidate's narrowed tree from them by the one function. +func TestAPreviewCarriesTheNamedPathsObjectsOfAFetchsModuleTrees(t *testing.T) { + var f Fetched + // As buildstep.ModuleTrees answers a module naming its paths: the named tree and the objects. + raw := `{"workspace":"w","merge":"6666666666666666666666666666666666666666","trees":{".":{"tree":"named:aa","manifest":"` + + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + `","paths":{"src":"1111111111111111111111111111111111111111"}}}}` + if err := json.Unmarshal([]byte(raw), &f); err != nil { + t.Fatal(err) + } + pv, ok := PreviewOf(Pull{Repository: "novox/mesh-sdk", Head: "h", Trunk: "t"}, []Fetched{f}) + if !ok || !reflect.DeepEqual(pv.Modules["."].Paths, map[string]string{"src": "1111111111111111111111111111111111111111"}) || pv.Modules["."].Tree != "named:aa" { + t.Errorf("preview = %+v, want the module's named tree and its objects", pv) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go index 874e99b..e4eebd3 100644 --- a/modules/mesh-delivery-checker/internal/check/read.go +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -7,6 +7,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the @@ -17,6 +18,9 @@ const ( VerdictsReadTool = "checker_verdicts" PullReadTool = "checker_pull" KeptReadTool = "checker_kept" + // PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685). + PreviewsReadTool = "checker_previews" + StatusReadTool = "checker_status" ) // StateRead reads one state: its keys and a key's stored value (nil for a key with none). @@ -26,7 +30,14 @@ type StateRead struct { } // Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates. -type Reader struct{ VerdictState, KeptState, CandidateState StateRead } +type Reader struct { + VerdictState, KeptState, CandidateState StateRead + + PreviewState StateRead // the checker's merge previews + PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent + StepStates map[string]StateRead // the build modules' steps states, by module + Capacity func() (string, error) // the capacity setting as written, `node=steps,...` +} // VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the // candidate still holds those heads; without it the report is that of the verdict alone. @@ -57,7 +68,7 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) { return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err) } c, ok := r.candidate(v.Group) - if !ok || c.HeadsHash != v.Heads { + if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) { out.Note = "the candidate no longer holds these heads" return out, nil } @@ -191,3 +202,153 @@ func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) { } return out, nil } + +// PreviewRead is one preview as stored, with the parts of its key. +type PreviewRead struct { + Key string `json:"key"` + Repository string `json:"repository"` + Head string `json:"head"` + Trunk string `json:"trunk"` + Preview json.RawMessage `json:"preview"` +} + +// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a +// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable. +func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) { + if r.PreviewState.Keys == nil || r.PreviewState.Get == nil { + return nil, nil, fmt.Errorf("the previews state cannot be read here") + } + keys, err := r.PreviewState.Keys() + if err != nil { + return nil, nil, err + } + sort.Strings(keys) + out := []PreviewRead{} + var bad []string + for _, k := range keys { + repo, h, trunk, err := delivery.ParsePreviewKey(k) + if err != nil { + bad = append(bad, k) + continue + } + if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) { + continue + } + raw, err := r.PreviewState.Get(k) + if err != nil { + return nil, nil, err + } + if len(raw) > 0 { + out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw}) + } + } + return out, bad, nil +} + +// RunningStep is a step in flight on a build module. +type RunningStep struct { + Module string `json:"module"` + Node string `json:"node"` + ID string `json:"id"` + Tool string `json:"tool"` + Since string `json:"since,omitempty"` +} + +// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in +// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a +// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters. +type Status struct { + Capacity map[string]int `json:"capacity"` + CapacityNote string `json:"capacity_note,omitempty"` + DefaultPerNode int `json:"default_per_node"` + PlansBuilding struct { + Readable bool `json:"readable"` + Keys []string `json:"keys"` + Holds bool `json:"holds"` + } `json:"plans_building"` + Running []RunningStep `json:"running_steps"` + RunningByNode map[string]int `json:"running_by_node"` + Unjudged []string `json:"unjudged_candidates"` + Notes []string `json:"notes,omitempty"` +} + +// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts. +func (r Reader) Status() (Status, error) { + st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}} + if r.Capacity == nil { + st.CapacityNote = "the capacity setting cannot be read from here" + } else if setting, err := r.Capacity(); err != nil { + st.CapacityNote = "the capacity setting cannot be read: " + err.Error() + } else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil { + st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error() + } + if r.PlansState.Keys == nil { + st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker") + } else if keys, err := r.PlansState.Keys(); err != nil { + st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error()) + } else { + sort.Strings(keys) + st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0 + } + mods := make([]string, 0, len(r.StepStates)) + for m := range r.StepStates { + mods = append(mods, m) + } + sort.Strings(mods) + st.Running = []RunningStep{} + for _, m := range mods { + sr := r.StepStates[m] + if sr.Keys == nil || sr.Get == nil { + continue + } + keys, err := sr.Keys() + if err != nil { + st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error()) + continue + } + sort.Strings(keys) + for _, k := range keys { + raw, err := sr.Get(k) + if err != nil || len(raw) == 0 { + continue + } + step, err := delivery.DecodeStep(raw) + if err != nil || step.State != delivery.StepRunning { + continue + } + rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool} + if !step.Started.IsZero() { + rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z") + } + st.Running = append(st.Running, rs) + st.RunningByNode[step.Node]++ + } + } + st.Unjudged = []string{} + if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil { + keys, err := r.CandidateState.Keys() + if err != nil { + return st, err + } + sort.Strings(keys) + for _, k := range keys { + raw, err := r.CandidateState.Get(k) + if err != nil || len(raw) == 0 { + continue + } + c, err := delivery.DecodeCandidate(raw) + if err != nil { + continue + } + vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) + if err != nil { + continue + } + if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 { + st.Unjudged = append(st.Unjudged, c.Group) + } + } + } + st.Notes = append(st.Notes, "the steps the checker holds waiting for a place are in its process and in no bucket: a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held") + return st, nil +} diff --git a/modules/mesh-delivery-checker/internal/check/read_test.go b/modules/mesh-delivery-checker/internal/check/read_test.go index 1d1c8fb..de12542 100644 --- a/modules/mesh-delivery-checker/internal/check/read_test.go +++ b/modules/mesh-delivery-checker/internal/check/read_test.go @@ -3,8 +3,10 @@ package check import ( "context" "encoding/json" + "os" "strings" "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -102,3 +104,64 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // t.Error("a digest nothing kept was answered") } } + +// 1680 live acceptance: checker_previews shows the preview key of an open pull request, as stored, narrowed by repository and head. +func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) { + pv := delivery.Preview{Repository: repoA, Head: head1, Trunk: head2, Merge: "m1", Modules: map[string]delivery.PreviewModule{}} + data, err := delivery.EncodePreview(pv) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.PreviewKey(repoA, head1, head2) + rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()} + got, bad, err := rd.Previews(repoA, head1[:8]) + if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) { + t.Fatalf("previews = %+v, %v", got, err) + } + if len(bad) != 1 || bad[0] != "junk" { + t.Errorf("unreadable = %v", bad) + } + if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 { + t.Errorf("another repository's = %+v", got) + } + if _, _, err := (Reader{}).Previews("", ""); err == nil { + t.Error("answered with no state to read") + } +} + +// 1685 live acceptance: checker_status reads the capacity, whether a plan key holds the checker, the steps in flight by node and +// the candidates without a verdict. +func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) { + running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)}) + passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed}) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + ckey, _ := delivery.CandidatesKey("feat/x") + rd := Reader{ + Capacity: func() (string, error) { return "shanks=2,ace=0", nil }, + PlansState: fakeState{"p1": []byte("{}")}.read(), + StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()}, + CandidateState: fakeState{ckey: cand}.read(), + VerdictState: fakeState{}.read(), + } + st, err := rd.Status() + if err != nil { + t.Fatal(err) + } + if st.Capacity["shanks"] != 2 || st.Capacity["ace"] != 0 || st.DefaultPerNode != 1 { + t.Errorf("capacity = %v default %d", st.Capacity, st.DefaultPerNode) + } + if !st.PlansBuilding.Holds || len(st.PlansBuilding.Keys) != 1 { + t.Errorf("plans = %+v", st.PlansBuilding) + } + if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 { + t.Errorf("running = %+v %v", st.Running, st.RunningByNode) + } + if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" { + t.Errorf("unjudged = %v", st.Unjudged) + } + // An absent plan bucket is read as empty and said; an unreadable capacity is said, not guessed. + st, err = Reader{Capacity: func() (string, error) { return "", os.ErrNotExist }}.Status() + if err != nil || st.PlansBuilding.Holds || st.CapacityNote == "" || len(st.Notes) == 0 { + t.Errorf("empty status = %+v, %v", st, err) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/results.go b/modules/mesh-delivery-checker/internal/check/results.go index 5583592..07be860 100644 --- a/modules/mesh-delivery-checker/internal/check/results.go +++ b/modules/mesh-delivery-checker/internal/check/results.go @@ -10,12 +10,15 @@ import ( // cmd/mesh-build-node check.go TestResult, fetch.go FetchResult) with the fields the fetch step adds for a merge (ADR 0484 // clause 3). Only the fields the checker reads are named; the rest is ignored. -// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest and the -// hashes of its lock files by name. A bare string is the tree alone. +// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest, the +// hashes of its lock files by name and the objects of its named paths. A bare string is the tree alone. type MergeTree struct { Tree string `json:"tree"` Manifest string `json:"manifest,omitempty"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's object; Tree is then their named + // tree (mesh-sdk buildstep.ModuleTree, novox/hq ADR 0499 clause 2). + Paths map[string]string `json:"paths,omitempty"` } // UnmarshalJSON takes a string as the tree alone, or an object. @@ -89,6 +92,9 @@ type Tested struct { // Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import // path. type Digested struct { + // Beside are the lines ` ` of the siblings the digest folded in (DigestVersion 3, lead decision L5), + // sorted; absent when the workspace has none or the build module folds none in. + Beside []string `json:"beside,omitempty"` Level string `json:"level"` Packages map[string]string `json:"packages"` WithoutTests []string `json:"without_tests,omitempty"` diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go new file mode 100644 index 0000000..ac5c6db --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -0,0 +1,389 @@ +package check + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501 +// §5; issue 686's `mesh-controller.try`, which calls these tools from R4e): the head merged onto its trunk, the unit and the +// integration level through the build modules. It owes the forge nothing (no status, no preview) and writes no verdict; its +// state is the value of its key in the checker's state tries. + +// The try's tools and state. +const ( + TryTool = "checker_try" + TryReadTool = "checker_try_read" + // TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-. + TriesState = "tries" + TriesBucket = delivery.CheckerModule + "_" + TriesState + // MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes. + MaxTries = 4 + // TryKeep is how long an ended try's key is kept before the checker removes it. + TryKeep = 7 * 24 * time.Hour +) + +// The states of a try: asked and waiting, running, and its end. +const ( + TryQueued = "queued" + TryRunning = "running" + TryPass = "pass" + TryFail = "fail" + TryError = "error" +) + +var ( + tryKeyOK = regexp.MustCompile(`^try-[0-9]{1,20}$`) + tryHeadOK = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) + tryTestsOK = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`) + tryRepoOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + tryTrunkOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`) +) + +// Try is the value of a tries key. +type Try struct { + V int `json:"v"` + ID string `json:"id"` + Repository string `json:"repository"` + // Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name. + Head string `json:"head"` + Trunk string `json:"trunk"` + Branch string `json:"branch,omitempty"` + // Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that + // level runs. Empty is every test of both levels. + Tests string `json:"tests,omitempty"` + Level string `json:"level,omitempty"` + State string `json:"state"` + // Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part. + Said string `json:"said,omitempty"` + // Took is the time from its start to its end, as Go writes a duration. + Took string `json:"took,omitempty"` + Failed []string `json:"failed,omitempty"` + Levels map[string]delivery.LevelResult `json:"levels,omitempty"` + Machines []string `json:"machines,omitempty"` + Asked time.Time `json:"asked"` + Started time.Time `json:"started,omitzero"` + Ended time.Time `json:"ended,omitzero"` +} + +// Over says whether the try came to its end. +func (t Try) Over() bool { return t.State == TryPass || t.State == TryFail || t.State == TryError } + +// DecodeTry reads a tries value. +func DecodeTry(raw []byte) (Try, error) { + var t Try + if err := json.Unmarshal(raw, &t); err != nil { + return Try{}, fmt.Errorf("not a try: %w", err) + } + if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" { + return Try{}, errors.New("not a try: it names no id, repository, head or trunk") + } + return t, nil +} + +func encodeTry(t Try) ([]byte, error) { + t.V = delivery.Version + return json.Marshal(t) +} + +// TryAsk is what checker_try is given. +type TryAsk struct { + Repository, Head, Trunk, Tests, Level string +} + +// TryStore is the tries state as the tools see it. +type TryStore interface { + Keys() ([]string, error) + Get(key string) (json.RawMessage, error) + Create(key string, value []byte) error +} + +// NotChecked is the error of a try of a repository the checker judges no level of: hq, judged by merge-check.sh (ADR 0484 +// clause 8), and a repository whose trunk the gate's source does not name. +func NotChecked(repository string) error { + return fmt.Errorf("no level of %s is checked by the checker", repository) +} + +// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second +// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a +// repository with none, and hq, are no repository the checker checks. A trunk asked by name is its trunk head now. +func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) { + repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git")) + head := strings.ToLower(strings.TrimSpace(a.Head)) + switch { + case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."): + return "", fmt.Errorf("%q is not a repository, /", a.Repository) + case !tryHeadOK.MatchString(head): + return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head) + case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration: + return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level) + case a.Tests != "" && !tryTestsOK.MatchString(a.Tests): + return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests) + } + trunkHead := "" + for r, h := range trunks { + if strings.EqualFold(r, repo) { + trunkHead = h + } + } + if repo == HQRepository || trunkHead == "" { + return "", NotChecked(repo) + } + t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()} + switch trunk := strings.TrimSpace(a.Trunk); { + case trunk == "": + t.Trunk = trunkHead + case tryHeadOK.MatchString(strings.ToLower(trunk)): + t.Trunk = strings.ToLower(trunk) + case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."): + // The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now. + t.Trunk, t.Branch = trunkHead, trunk + default: + return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk) + } + keys, err := store.Keys() + if err != nil { + return "", fmt.Errorf("the tries cannot be listed: %w", err) + } + sort.Strings(keys) + var open []string + for _, k := range keys { + raw, err := store.Get(k) + if err != nil || len(raw) == 0 { + continue + } + cur, err := DecodeTry(raw) + if err != nil || cur.Over() { + continue + } + if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level { + return cur.ID, nil + } + open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)) + } + if len(open) >= MaxTries { + return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", ")) + } + t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10) + data, err := encodeTry(t) + if err != nil { + return "", err + } + if err := store.Create(t.ID, data); err != nil { + return "", fmt.Errorf("the try %s cannot be written: %w", t.ID, err) + } + return t.ID, nil +} + +// ReadTry is a try's value, as stored. +func ReadTry(store TryStore, id string) (json.RawMessage, error) { + if !tryKeyOK.MatchString(id) { + return nil, fmt.Errorf("%q is not a try's id, try-", id) + } + raw, err := store.Get(id) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep) + } + return raw, nil +} + +// TryOptions narrow what a try runs; a pull request's check has none. +type TryOptions struct { + Level string + Run string +} + +// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check, and writes its +// end. It holds no timer: a try starts when its key is written, or when a delivery that held it ends (Reconsider). +type Tryer struct { + Tries KV + Judge Judge + // Hold says whether a delivery builds, so no try starts (ADR 0484 clause 12); nil never holds. + Hold func() bool + Bound time.Duration + Now func() time.Time + Logf func(format string, args ...any) + + mu sync.Mutex + wg sync.WaitGroup + inflight map[string]bool + seen map[string]bool // looked at since this process started: a try left running is taken over the first time +} + +func (tr *Tryer) now() time.Time { + if tr.Now != nil { + return tr.Now() + } + return time.Now() +} + +func (tr *Tryer) logf(format string, args ...any) { + if tr.Logf != nil { + tr.Logf(format, args...) + } +} + +// Wait returns when every try started so far has ended. +func (tr *Tryer) Wait() { tr.wg.Wait() } + +// Put takes a value of the tries state in: a queued try is started, and a try left running is taken over the first time +// this process sees it (the checker that ran it stopped; there is one checker). +func (tr *Tryer) Put(ctx context.Context, key string, value []byte) { + t, err := DecodeTry(value) + if err != nil { + tr.logf("tries.%s holds what is not a try, left: %v", key, err) + return + } + if t.Over() { + return + } + held := tr.Hold != nil && tr.Hold() + tr.mu.Lock() + if tr.inflight == nil { + tr.inflight, tr.seen = map[string]bool{}, map[string]bool{} + } + if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) { + tr.mu.Unlock() + return + } + if held { + tr.mu.Unlock() // not seen yet: a try left running is still taken over when the hold ends + tr.logf("%s: not started, a delivery builds", key) + return + } + tr.seen[key] = true + tr.inflight[key] = true + tr.mu.Unlock() + tr.wg.Add(1) + go func() { + defer tr.wg.Done() + defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }() + tr.run(ctx, key) + }() +} + +// Reconsider looks at every try again (a delivery that held them ended), and removes the ended tries older than TryKeep. +func (tr *Tryer) Reconsider(ctx context.Context) { + keys, err := tr.Tries.Keys(ctx) + if err != nil { + tr.logf("the tries cannot be listed: %v", err) + return + } + for _, k := range keys { + raw, _, found, err := tr.Tries.Get(ctx, k) + if err != nil || !found { + continue + } + if t, err := DecodeTry(raw); err == nil && t.Over() && tr.now().Sub(t.Ended) > TryKeep { + if d, ok := tr.Tries.(interface { + Delete(ctx context.Context, key string) error + }); ok { + if err := d.Delete(ctx, k); err != nil { + tr.logf("%s ended %s ago and could not be removed: %v", k, tr.now().Sub(t.Ended).Round(time.Hour), err) + } + } + continue + } + tr.Put(ctx, k, raw) + } +} + +// run claims the try by compare-and-set, judges it within its bound and writes its end. +func (tr *Tryer) run(ctx context.Context, key string) { + raw, rev, found, err := tr.Tries.Get(ctx, key) + if err != nil || !found { + return + } + t, err := DecodeTry(raw) + if err != nil || t.Over() { + return + } + t.State, t.Started = TryRunning, tr.now().UTC() + data, err := encodeTry(t) + if err != nil { + return + } + if rev, err = tr.Tries.Update(ctx, key, data, rev); err != nil { + if !errors.Is(err, ErrConflict) { + tr.logf("%s: %v", key, err) + } + return + } + cs := &Case{Key: key, Group: key, + Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}}, + Trunk: map[string]string{t.Repository: t.Trunk}, + Queued: t.Asked, + Progress: &Progress{}, + Try: &TryOptions{Level: t.Level, Run: t.Tests}} + out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound) + if ctx.Err() != nil { + return // left running: the checker takes it over when it starts + } + v := out.Verdict + t.Ended = tr.now().UTC() + t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() + t.Levels, t.Machines = v.Levels, v.Machines + t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest) + switch { + case !judged || IsNodesFailure(v): + t.State = TryError + case v.State == delivery.VerdictPass: + t.State = TryPass + default: + t.State = TryFail + if f := firstFailed(v); f != "" { + t.Failed = []string{f} + } + } + if t.Tests != "" { + t.Said += " | only the tests " + t.Tests + " (Go); nothing kept" + } + if data, err = encodeTry(t); err == nil { + _, err = tr.Tries.Update(ctx, key, data, rev) + } + if err != nil { + tr.logf("%s: its end could not be written: %v", key, err) + return + } + tr.logf("%s: %s", key, t.State) +} + +// judgeWithin runs a case within the bound (DefaultBound when 0). A judge that fails to judge, and a check that ran past its +// bound, are verdicts of failure naming the machine and the step, and judged says false for them. +func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) { + if bound <= 0 { + bound = DefaultBound + } + run, cancel := context.WithTimeoutCause(ctx, bound, errBound) + defer cancel() + out, err := j.Judge(run, cs) + stage, node := cs.Progress.Now() + if node == "" { + node = "checker" + } + switch { + case ctx.Err() != nil: + return Outcome{}, false + case context.Cause(run) == errBound: + if stage == "" { + stage = "starting" + } + return Outcome{Verdict: failure(node, fmt.Sprintf("the check ran past its bound of %s, stuck at: %s", bound, stage))}, false + case err != nil: + return Outcome{Verdict: failure(node, err.Error())}, false + } + return out, true +} diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go new file mode 100644 index 0000000..65b89b5 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -0,0 +1,196 @@ +package check + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// memTries is the tries state as the tools see it, over a memKV. +type memTries struct{ kv *memKV } + +func (m memTries) Keys() ([]string, error) { return m.kv.Keys(context.Background()) } +func (m memTries) Get(key string) (json.RawMessage, error) { + v, _, _, err := m.kv.Get(context.Background(), key) + return v, err +} +func (m memTries) Create(key string, value []byte) error { + _, err := m.kv.Create(context.Background(), key, value) + return err +} + +var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, HQRepository: trunkA} + +func readTry(t testing.TB, kv *memKV, id string) Try { + t.Helper() + raw, _, found, _ := kv.Get(context.Background(), id) + if !found { + t.Fatalf("no try %s", id) + } + tr, err := DecodeTry(raw) + if err != nil { + t.Fatal(err) + } + return tr +} + +func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR 0501 §5 + kv := newMemKV() + store := memTries{kv} + at := time.Date(2026, 10, 22, 18, 0, 0, 0, time.UTC) + id, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: "main"}, at) + if err != nil || !strings.HasPrefix(id, "try-") { + t.Fatalf("id %q, %v", id, err) + } + tr := readTry(t, kv, id) + if tr.State != TryQueued || tr.Trunk != trunkA || tr.Branch != "main" || tr.Head != head1 || tr.Repository != repoA { + t.Errorf("try = %+v, want queued on the trunk's head", tr) + } + again, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: "main"}, at.Add(time.Second)) + if err != nil || again != id { + t.Errorf("a second ask of the same head answered %q, %v, want %q", again, err, id) + } + heads := []string{head2, trunkA2, trunkS2} + for i, h := range heads { + if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: h, Trunk: trunkA}, at.Add(time.Duration(i+2)*time.Second)); err != nil { + t.Fatalf("try %d: %v", i+2, err) + } + } + if _, err := AskTry(store, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute)); err == nil || !strings.Contains(err.Error(), "4 tries wait or run") { + t.Errorf("a fifth try answered %v, want refused naming the four", err) + } +} + +func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agreed error of the try + store := memTries{newMemKV()} + for _, repo := range []string{HQRepository, "novox/mesh-lab"} { + _, err := AskTry(store, tryTrunks, TryAsk{Repository: repo, Head: head1, Trunk: "main"}, time.Now()) + if err == nil || err.Error() != "no level of "+repo+" is checked by the checker" { + t.Errorf("%s: %v", repo, err) + } + } + for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"}, + {Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} { + if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil { + t.Errorf("%+v was taken", bad) + } + } +} + +func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5 + ctx := context.Background() + kv := newMemKV() + id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now()) + if err != nil { + t.Fatal(err) + } + b := newBench(t) + tr := &Tryer{Tries: kv, Judge: b.ex} + raw, _, _, _ := kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + got := readTry(t, kv, id) + if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 { + t.Fatalf("try = %+v", got) + } + fetch := b.steps.tool("fetch") + if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" { + t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch) + } + if keys, _ := b.prev.Keys(ctx); len(keys) != 0 { + t.Errorf("a try wrote the previews %v", keys) + } + // An ended try is not run again. + raw, _, _, _ = kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + if n := len(b.steps.tool("fetch")); n != 1 { + t.Errorf("an ended try was fetched again: %d fetches", n) + } +} + +func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { + ctx := context.Background() + kv := newMemKV() + id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelIntegration, Tests: "TestBus|TestStore"}, time.Now()) + if err != nil { + t.Fatal(err) + } + b := newBench(t) + tr := &Tryer{Tries: kv, Judge: b.ex} + raw, _, _, _ := kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + tests := b.steps.tool("test") + if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" { + t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests) + } + if n := len(b.steps.tool("digest")); n != 0 { + t.Errorf("%d digests for a try of some tests", n) + } + if keys, _ := b.kept.Keys(ctx); len(keys) != 0 { + t.Errorf("kept %v from a try of some tests", keys) + } + if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") { + t.Errorf("try = %+v", got) + } +} + +func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) { + ctx := context.Background() + kv := newMemKV() + b := newBench(t) + b.test = func(c stepCall) (any, error) { return captured[Tested](t, "go-test-failing"), nil } + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) + tr := &Tryer{Tries: kv, Judge: b.ex} + raw, _, _, _ := kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], "TestThreeIsFour") { + t.Errorf("try = %+v", got) + } + // No build node: the try could not run, which is an error, not a failure of the change. + kv2 := newMemKV() + b2 := newBench(t) + b2.ex.Toolchains = entries{} + id2, _ := AskTry(memTries{kv2}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) + tr2 := &Tryer{Tries: kv2, Judge: b2.ex} + raw, _, _, _ = kv2.Get(ctx, id2) + tr2.Put(ctx, id2, raw) + tr2.Wait() + if got := readTry(t, kv2, id2); got.State != TryError { + t.Errorf("try with no node = %+v, want error", got) + } +} + +func TestATryWaitsWhileADeliveryBuildsAndATryLeftRunningIsTakenOver(t *testing.T) { + ctx := context.Background() + kv := newMemKV() + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) + // Left running by a checker that stopped. + raw, rev, _, _ := kv.Get(ctx, id) + cur, _ := DecodeTry(raw) + cur.State = TryRunning + data, _ := encodeTry(cur) + if _, err := kv.Update(ctx, id, data, rev); err != nil { + t.Fatal(err) + } + b := newBench(t) + held := true + tr := &Tryer{Tries: kv, Judge: b.ex, Hold: func() bool { return held }} + tr.Reconsider(ctx) + tr.Wait() + if n := len(b.steps.tool("fetch")); n != 0 { + t.Fatalf("a try ran while a delivery builds") + } + held = false + tr.Reconsider(ctx) + tr.Wait() + if got := readTry(t, kv, id); got.State != TryPass { + t.Errorf("the try left running = %+v, want taken over and passed", got) + } +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index c3d4592..a4b5442 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -92,7 +92,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("module = %q", m.Module) } // delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets. - if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", @@ -118,7 +118,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("streams = %+v, want %+v", m.Streams, want) } // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. - if want := []string{"checker_kept", "checker_pull", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + // The try's two tools (ADR 0501 §5) beside them: mesh-controller's try calls them. + if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 87c1005..2c22576 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -13,6 +13,7 @@ { "name": "kept", "history": 1 }, { "name": "owed", "history": 5 }, { "name": "previews", "history": 5 }, + { "name": "tries", "history": 1 }, { "name": "verdicts", "history": 5 } ], "reads": [ @@ -31,7 +32,11 @@ ], "tools": [ "checker_kept", + "checker_previews", "checker_pull", + "checker_status", + "checker_try", + "checker_try_read", "checker_verdicts" ], "invokes": [ diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go index 410ed63..78ac361 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go @@ -96,27 +96,6 @@ func (p *performer) owed(ctx context.Context, key string, rev uint64, value []by p.perform(ctx, "owed."+key, key, rev, value) } -// checkOwed performs the status the checker owes on a head (mesh-delivery-checker_owed; novox/hq ADR 0484 clause 2) at -// revision rev, once. Its keys are kept apart from the gate's, and a key that is not the checker's one act, the status -// mesh/check, is left. -func (p *performer) checkOwed(ctx context.Context, key string, rev uint64, value []byte) { - doneKey := "check." + key - if p.seen(ctx, doneKey, rev) { - return - } - if _, _, _, err := delivery.ParseCheckerOwedKey(key); err != nil { - p.logger.Printf("%s at revision %d is not a status of the checker, and is left: %v", key, rev, err) - p.keep(ctx, doneKey, rev) - return - } - if _, err := delivery.DecodeCheckerOwed(value); err != nil { - p.logger.Printf("%s at revision %d is not a status of the checker, and is left: %v", key, rev, err) - p.keep(ctx, doneKey, rev) - return - } - p.perform(ctx, doneKey, key, rev, value) -} - // perform does the act of an owed key once for the revision, under the bookkeeping key doneKey. func (p *performer) perform(ctx context.Context, doneKey, key string, rev uint64, value []byte) { if p.seen(ctx, doneKey, rev) { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 2a7b64b..608685e 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -208,57 +208,3 @@ func TestAMergeWithTwoHandedStopsAtTheFirstRefusal(t *testing.T) { // ADR 0441 t.Errorf("calls %d messages %d, want the first refusal to stop the pass", len(f.calls), len(pub.data)) } } - -// checkerOwed is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed). -func checkerOwed(t *testing.T, number int, state, text string) (string, []byte) { - t.Helper() - key, err := delivery.CheckerOwedKey("novox/mesh-delivery", number, headA) - if err != nil { - t.Fatal(err) - } - v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Text: text, Target: "https://view.example/checks/g.0123456789ab"}) - if err != nil { - t.Fatal(err) - } - return key, v -} - -func TestTheCheckersStatusIsSetOncePerRevisionAndAKeyOfTheGateIsNotItsKey(t *testing.T) { // 1683/AC5 (ADR 0484 clause 2) - f := &forgeFake{} - p, _, done, _ := newPerformer(f) - key, v := checkerOwed(t, 12, "success", "pass aaaaaaaaaaaa on bbbbbbbbbbbb") - p.checkOwed(context.Background(), key, 3, v) - p.checkOwed(context.Background(), key, 3, v) - if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_commit_status" || - f.calls[0].body != `{"context":"mesh/check","description":"pass aaaaaaaaaaaa on bbbbbbbbbbbb","owner":"novox","repo":"mesh-delivery","sha":"`+headA+`","state":"success","target_url":"https://view.example/checks/g.0123456789ab"}` { - t.Fatalf("calls = %+v", f.calls) - } - p.checkOwed(context.Background(), key, 4, v) - if len(f.calls) != 2 { - t.Errorf("a new revision was not performed: %+v", f.calls) - } - if _, ok := done["check."+key]; !ok { - t.Errorf("revision kept under %v, want check.%s", done, key) - } - // The gate's own owed bookkeeping is another key: the same owed key seen in the gate's bucket is its own act. - if _, ok := done["owed."+key]; ok { - t.Errorf("the checker's status was kept under the gate's key: %v", done) - } -} - -func TestAnythingButTheCheckersStatusInItsBucketIsLeft(t *testing.T) { // ADR 0484 clause 2: the checker owes one act - f := &forgeFake{} - p, _, done, logs := newPerformer(f) - // A comment, and a status of another context: the SDK refuses both to the checker's encoder, so they are written raw. - commentKey, commentValue := owedKV(t, "novox/mesh-delivery", 12, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "hello"}) - gateKey, gateValue := owedKV(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", - delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "success", Text: "x", Target: "https://x.example"}) - p.checkOwed(context.Background(), commentKey, 1, commentValue) - p.checkOwed(context.Background(), gateKey, 1, gateValue) - if len(f.calls) != 0 { - t.Errorf("the forge was called: %+v", f.calls) - } - if !strings.Contains(logs.String(), commentKey) || !strings.Contains(logs.String(), gateKey) || len(done) != 2 { - t.Errorf("log %q, kept %v: both keys are named and kept so they are not tried again", logs.String(), done) - } -} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 679a8ae..05497d4 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -30,9 +30,8 @@ import ( // The gate's buckets mesh-delivery-gate_merges and mesh-delivery-gate_owed are read and watched (the manifest's reads), the // bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342). // -// The checker's owed bucket is read like the gate's (reads) and the checker's stream checks is heard like EVENTS is (consumes -// mesh-delivery-checker.owed.written; ADR 0484 clause 2): those entries are written by analogy with the ones the controller -// composes for the gate's buckets and for the EVENTS consumer, not taken from a composed file. +// The bridge neither reads the checker's owed nor hears its stream (ADR 0521, R4e finding R2-1): the forge module writes +// mesh/check from it. The stream checks is made here so a test can say the checker wrote a status. // // The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal // with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls, @@ -108,14 +107,6 @@ accounts { "mesh.mod.gitea.tool.gitea_add_comment" "mesh.mod.gitea.tool.gitea_add_comment.*" "mesh.stream.mesh-delivery-forge-bridge.event.forge.>" - "$JS.ACK.CHECKS.n1_mesh-delivery-forge-bridge.>" - "$JS.API.CONSUMER.INFO.CHECKS.n1_mesh-delivery-forge-bridge" - "$JS.API.CONSUMER.MSG.NEXT.CHECKS.n1_mesh-delivery-forge-bridge" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-checker_owed.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-checker_owed.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-checker_owed.>" - "$JS.API.STREAM.INFO.KV_mesh-delivery-checker_owed" - "$JS.FC.KV_mesh-delivery-checker_owed.>" ] } subscribe { allow: [ "_INBOX.n1.mesh-delivery-forge-bridge.>" @@ -217,15 +208,10 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil { t.Fatal(err) } - // The checker's stream checks and the bridge's durable consumer on its owed.written event; the checker's owed bucket is made - // only by the test that needs it, as when the checker is assigned after the bridge started. + // The checker's stream checks, which the bridge no longer hears (ADR 0521): a test says on it that the checker wrote a status. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil { t.Fatal(err) } - if _, err := js.CreateOrUpdateConsumer(ctx, "CHECKS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-forge-bridge", AckPolicy: jetstream.AckExplicitPolicy, - FilterSubjects: []string{"mesh.stream.mesh-delivery-checker.event.owed.written"}}); err != nil { - t.Fatal(err) - } // The bridge's state bucket, made by the controller from the manifest's state entry before the process starts. if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil { t.Fatal(err) @@ -728,16 +714,17 @@ func forgeMerged(t *testing.T, b *bed) []string { func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] } -// TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey: the checker's owed bucket is made after the bridge started; -// the checker writes the status mesh/check it owes on a head (the SDK's CheckerOwedKey and EncodeCheckerOwed, which the checker -// itself writes with) and says so on its stream; the bridge sets the status on the forge within 10 s, with the context -// mesh/check, the state and the line; the same revision is not set again, a new revision is (ADR 0484 clause 2). -func TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey(t *testing.T) { // 1683/AC5, ADR 0484 §2 +// TestBridgePerformsNoMeshCheckAct: the checker owes mesh/check on a head and says so on its stream; the bridge calls no +// tool with mesh/check and publishes nothing of its own on EVENTS for it. The forge module writes mesh/check from the +// checker's owed state itself (ADR 0521; R4e round 2 finding R2-1). +func TestBridgePerformsNoMeshCheckAct(t *testing.T) { // R2-1, ADR 0521 b := startBed(t, `{"result":{"pulls":[]}}`, nil) calls := make(chan string, 8) - sub, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_status", func(m *nats.Msg) { - calls <- string(m.Data) - _ = m.Respond([]byte(`{"result":{},"node":"n0"}`)) + // Every forge call is watched, and none answered here: the bed's stand-ins answer the ones the bridge asks. + sub, err := b.ctl.Subscribe("mesh.mod.gitea.>", func(m *nats.Msg) { + if strings.Contains(string(m.Data), "mesh/check") { + calls <- m.Subject + " " + string(m.Data) + } }) if err != nil { t.Fatal(err) @@ -754,38 +741,26 @@ func TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey(t *testing.T if err != nil { t.Fatal(err) } - put := func(state, text string) { - v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Text: text, Target: "https://view.example/checks/g.0123456789ab"}) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put(b.ctx, key, v); err != nil { - t.Fatal(err) - } - if _, err := b.js.Publish(b.ctx, owedWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil { - t.Fatal(err) - } + v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.0123456789ab"}) + if err != nil { + t.Fatal(err) } - next := func() string { - select { - case got := <-calls: - return got - case <-time.After(10 * time.Second): - t.Fatal("the status was not set within 10 s") - } - return "" + if _, err := kv.Put(b.ctx, key, v); err != nil { + t.Fatal(err) } - put("pending", "checking 0123456789ab on aaaaaaaaaaaa") - if got := next(); !strings.Contains(got, `"context":"mesh/check"`) || !strings.Contains(got, `"state":"pending"`) || !strings.Contains(got, `"sha":"`+head+`"`) || !strings.Contains(got, `"repo":"mesh-delivery"`) { - t.Fatalf("first call = %s", got) - } - put("success", "pass 0123456789ab on aaaaaaaaaaaa | unit 2 run, 3 kept") - if got := next(); !strings.Contains(got, `"state":"success"`) || !strings.Contains(got, "unit 2 run, 3 kept") { - t.Fatalf("second call = %s", got) + if _, err := b.js.Publish(b.ctx, "mesh.stream.mesh-delivery-checker.event.owed.written", []byte(`{"key":"`+key+`"}`)); err != nil { + t.Fatal(err) } select { - case extra := <-calls: - t.Errorf("a third call was made: %s", extra) - case <-time.After(500 * time.Millisecond): + case c := <-calls: + t.Errorf("a forge tool was called with mesh/check: %s", c) + case <-time.After(2 * time.Second): + } + events, err := b.js.Stream(b.ctx, "EVENTS") + if err != nil { + t.Fatal(err) + } + if _, err := events.GetLastMsgForSubject(b.ctx, "mesh.mod.mesh-delivery-forge-bridge.event.>"); err == nil { + t.Error("the bridge published an event of its own for the checker's mesh/check") } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go deleted file mode 100644 index c25f87a..0000000 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "log" - "strings" - "sync" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// checksStream is the stream the checker carries its events on, as the bus names it (declared by the checker as the stream -// checks, heard by this module through consumes; novox/hq ADR 0484 clause 2). -const checksStream = "CHECKS" - -// owedWrittenSubject is the subject of the checker's owed.written event. -const owedWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.owed.written" - -// lateWatch watches a bucket that may be made after the bridge started: the checker's owed bucket. The bridge holds no timer -// (ADR 0437 §8), so it watches the bucket where it exists at start, and otherwise when the checker says it wrote a status -// (owed.written, handled by Written). -type lateWatch struct { - ctx context.Context - js interface { - KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) - } - bucket string - react func(ctx context.Context, key string, rev uint64, value []byte) - logger *log.Logger - errs chan error - mu sync.Mutex - watched bool -} - -// ensure starts the watch if the bucket exists and is not watched. -func (w *lateWatch) ensure() error { - w.mu.Lock() - defer w.mu.Unlock() - if w.watched { - return nil - } - kv, err := w.js.KeyValue(w.ctx, w.bucket) - switch { - case errors.Is(err, jetstream.ErrBucketNotFound): - w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a status", w.bucket) - return nil - case err != nil: - return fmt.Errorf("the bucket %s: %w", w.bucket, err) - } - w.watched = true - go func() { - w.errs <- watch(w.ctx, kv, func(key string, rev uint64, value []byte) { - hctx, cancel := context.WithTimeout(w.ctx, syncDeadline) - defer cancel() - w.react(hctx, key, rev, value) - }) - }() - return nil -} - -// Written takes one owed.written event of the checker in: its bucket is watched if it is not. A subject that is not the -// checker's is terminated; a failure stays unacknowledged, and the server delivers it again. -func (w *lateWatch) Written(m interface { - Subject() string - Ack() error - Term() error -}) { - if !strings.HasPrefix(m.Subject(), owedWrittenSubject) { - w.logger.Printf("%s: not the checker's owed.written, dropped", m.Subject()) - _ = m.Term() - return - } - if err := w.ensure(); err != nil { - w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) - return - } - _ = m.Ack() -} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index fc1aaff..a11a3b2 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -225,24 +225,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) }() } - // The checker's statuses (mesh/check, unrequired until release R4f): its bucket is made when the checker is assigned, which - // may be after the bridge started, and the bridge holds no timer; the checker says on its stream each time it wrote a - // status, and the bucket is then watched (late.go). - late := &lateWatch{ctx: ctx, js: js, bucket: delivery.CheckerOwedBucket, react: perf.checkOwed, logger: logger, errs: watching} - if err := late.ensure(); err != nil { - return err - } - switch ccons, err := js.Consumer(ctx, checksStream, cred.Node+"_"+cred.Module); { - case err != nil: - logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's statuses are not heard until the bridge starts again: %v", cred.Node, cred.Module, checksStream, err) - default: - cc, err := ccons.Consume(func(m jetstream.Msg) { late.Written(m) }, - jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) - if err != nil { - return err - } - defer cc.Stop() - } if err := said(ctx); err != nil { return err } diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 383758b..669a04d 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 3515fb8..db1ea18 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go index f844717..f35ce98 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go @@ -77,6 +77,12 @@ func OwedAct(key string, value []byte) (Act, error) { if o.Context != context { return Act{}, fmt.Errorf("the owed status %s says the context %q, not the %q its key names", key, o.Context, context) } + if o.Context == delivery.StatusContextCheck { + // No tool writes mesh/check (R4e finding K1), and the bridge performs no act of it: the forge module writes it from + // the checker's state mesh-delivery-checker.owed, read by a watch (ADR 0521, finding R2-1). + return Act{}, fmt.Errorf("the owed status %s is %s, which the bridge does not perform: the forge module writes it from mesh-delivery-checker.owed", + key, delivery.StatusContextCheck) + } a.Call = Call{Tool: commitStatusTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "context": o.Context, "state": o.State, "description": o.Text, "target_url": o.Target}} case delivery.OwedView: diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go index 5b555b8..4883012 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go @@ -2,6 +2,7 @@ package bridge import ( "reflect" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -43,6 +44,37 @@ func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1 } } +// checkerKV is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed). +func checkerKV(t *testing.T, o delivery.Owed) (string, []byte) { + t.Helper() + key, err := delivery.CheckerOwedKey("novox/mesh-delivery", 12, headA) + if err != nil { + t.Fatal(err) + } + v, err := delivery.EncodeCheckerOwed(o) + if err != nil { + t.Fatal(err) + } + return key, v +} + +// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no +// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge +// performs no mesh/check act. +func TestOwedMeshCheckIsNoToolCall(t *testing.T) { // K1 + key, v := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"}) + a, err := OwedAct(key, v) + if err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { + t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err) + } + // Nor through the gate's own owed bucket: the context is refused whichever key carries it. + gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check", + delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"}) + if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { + t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err) + } +} + func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2 cases := []struct { act string diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go new file mode 100644 index 0000000..57c06bc --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go @@ -0,0 +1,36 @@ +package bridge + +import ( + "os" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a +// module whose artifacts name their paths (mesh-sdk's module.json at 9cb4d59) gives a message that carries the manifest as +// the forge answered it and no named path's object: the indexer asks the forge for those itself. +func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) { + raw, err := os.ReadFile("testdata/mesh-sdk-9cb4d59.module.json") + if err != nil { + t.Fatal(err) + } + files := func(owner, repo, path, ref string) ([]byte, error) { + if path == "module.json" { + return raw, nil + } + return nil, ErrNotFound + } + var calls []call + msg, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}), treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), files) + if err != nil { + t.Fatal(err) + } + f, err := delivery.DecodeForge(msg.Data) + if err != nil { + t.Fatal(err) + } + if len(f.Paths) != 0 || f.Manifests[""] != string(raw) || len(calls) != 1 { + t.Errorf("message paths %v, manifest carried %v, trees asked %d times; want no paths, the manifest, one ask", f.Paths, f.Manifests[""] == string(raw), len(calls)) + } +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json b/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json new file mode 100644 index 0000000..404139a --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json @@ -0,0 +1,16 @@ +{ + "module": "mesh-sdk", + "version": "1", + "slug": "sdk", + "build": { + "artifacts": [ + { + "name": "lib", + "kind": "package", + "language": "typescript", + "paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"] + } + ] + }, + "resources": [] +} diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 40fb8a8..75e356b 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -86,10 +86,11 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**"}}}) { t.Errorf("streams = %+v", m.Streams) } - if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated", "mesh-delivery-checker.owed.written"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // The checker's owed is the forge module's to write (ADR 0521, R4e finding R2-1): the bridge neither hears nor reads it. + if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } - if want := []string{"mesh-delivery-checker.owed", "mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view", @@ -108,7 +109,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}}) { // 1279/AC6, 1341 t.Errorf("state = %+v, want the buckets heads and performed", m.State) } - // No other state, no bus, no seat, no tool (ADR 0437 §4). + // No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status). for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the bridge has none", k) diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index d411e5f..25b68be 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -5,8 +5,7 @@ "consumes": [ "gitea.pull.updated", "gitea.pull.merged", - "gitea.pull.closed", - "mesh-delivery-checker.owed.written" + "gitea.pull.closed" ], "streams": [ { "name": "forge", "events": ["forge.**"] } @@ -16,7 +15,6 @@ { "name": "performed", "history": 1 } ], "reads": [ - "mesh-delivery-checker.owed", "mesh-delivery-gate.merges", "mesh-delivery-gate.owed" ], diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index a885d9d..448e24c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -183,30 +184,24 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. -func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 - // Inputs and record in the controller's layout: SourceInputs under its Go field names (mesh-sdk - // go/delivery/testdata/build-record-build-agent.json, captured from the controller's encoder). - const inputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` +func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) + // The bucket holds only build-agent's record, the one the indexer reads until R4g (the builder's own input). js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("a", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("a"), []byte("build-1")); err != nil { + rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}` + if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil { t.Fatal(err) } }) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}}) + Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}, + Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}}) if err != nil { t.Fatal(err) } if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged", data); err != nil { t.Fatal(err) } - want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) var m delivery.Module for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if e, err := modules.Get(ctx, "a"); err == nil { @@ -215,8 +210,8 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 } } } - if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" { // 1606/AC2 - t.Errorf("module = %+v within 5 s, want fp %s and path modules/a", m, want) + if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.FP != "" || m.None != "" { // 1606/AC2 + t.Errorf("module = %+v within 5 s, want path modules/a, its manifest, and no value of today's form", m) } } @@ -440,3 +435,46 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc t.Errorf("none2 = %q within 5 s of the preview, want %q (computed from the preview)", got, delivery.ToolchainNoReport) } } + +// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59, +// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the +// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form +// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature). +func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) { + manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json") + if err != nil { + t.Fatal(err) + } + objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", + "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"} + js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) + merged := func(merge, root string) delivery.Module { + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70, + Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root}, + Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil { + t.Fatal(err) + } + var m delivery.Module + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := modules.Get(ctx, "mesh-sdk"); err == nil { + if m, err = delivery.DecodeModule(e.Value()); err == nil && m.Commit == merge { + return m + } + } + } + t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge) + return m + } + first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) { + t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2) + } + if first.FP2 != second.FP2 || first.None2 != second.None2 { + t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 54df569..d921599 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -25,6 +25,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) @@ -283,7 +284,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro errs := make(chan error, 1) go func() { errs <- follow(ctx, w, each) }() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf} + // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). + account, err := process.ReadAccount(path) + if err != nil { + return err + } + calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer calls.Close() + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, + Named: namedVia(calls.Call)} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go new file mode 100644 index 0000000..f8fd619 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory +// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each +// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk +// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). +const commitTreesTool = "gitea.gitea_commit_trees" + +// toolCaller calls a module's tool, `.`, as the indexer's own account (mesh-sdk process.Conn.Call). +type toolCaller func(key string, body any) (json.RawMessage, error) + +// namedVia asks the forge for the objects of the named paths through call. An answer without the object of every path +// asked is an error naming it. +func namedVia(call toolCaller) func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + return func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return nil, fmt.Errorf("%q is not /", repository) + } + raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths}) + if err != nil { + return nil, err + } + var a struct { + Paths map[string]map[string]string `json:"paths"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err) + } + for dir, want := range paths { + for _, p := range want { + if a.Paths[dir][p] == "" { + return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir) + } + } + } + return a.Paths, nil + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go new file mode 100644 index 0000000..9691911 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "encoding/json" + "testing" +) + +func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3 + var asked map[string]any + call := func(key string, body any) (json.RawMessage, error) { + if key != commitTreesTool { + t.Errorf("called %s", key) + } + asked = body.(map[string]any) + return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil + } + got, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}) + if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" { + t.Fatalf("objects = %v, %v", got, err) + } + if asked["owner"] != "novox" || asked["repo"] != "mesh-sdk" || asked["sha"] != "s" || asked["paths"] == nil { + t.Errorf("asked %v, want the owner, repo, commit and paths", asked) + } + call = func(string, any) (json.RawMessage, error) { + return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil + } + if _, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil { + t.Error("an answer missing a path asked was taken") + } +} diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index a97df59..6390b2e 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 3515fb8..db1ea18 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 72eec2a..5dfd681 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -1,31 +1,24 @@ -// Package index keeps the candidate groups (novox/hq ADR 0437 §5) the forge bridge's pull messages make. package index import ( "encoding/json" - "path" - "sort" "strings" "sync" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// Builds holds, from the controller's builds bucket (mesh-controller_builds), which module a directory of a -// repository is: the module of the newest build record naming that repository and path (BuildRecord.Path). Safe for use -// from a watch and from the message handler at once. +// Builds holds the controller's build records (mesh-controller_builds) for the two things the indexer still reads from +// them until release R4g, when build-agent goes (the lead's decision of R4e, 2026-10-22): +// - the Builder input of the builder's own module, which build-agent builds (TrunkOf; novox/hq ADR 0453 clause 5, +// issue 1727); +// - the directory of a module whose modules key has none (PathOf; issue 1614), filled once and changing no fingerprint. +// +// It lists no module and maps no directory to a module: the indexer's modules are those its own messages and bucket name +// (ADR 0501 clause 6, issue 1851). Safe for use from a watch and from the message handler at once. type Builds struct { mu sync.Mutex records map[string]delivery.BuildRecord // by bucket key - byDir map[string]delivery.BuildRecord // by repository and path; nil when stale - mods map[string]*moduleBuilds // by module; nil when stale -} - -// moduleBuilds is what the builds say of one module: its newest record of any kind, and its newest successful trunk -// build that recorded its inputs. -type moduleBuilds struct { - any delivery.BuildRecord - trunk *delivery.BuildRecord } // NewBuilds is an empty Builds. @@ -44,101 +37,19 @@ func (b *Builds) Put(key string, value []byte) { b.mu.Lock() defer b.mu.Unlock() b.records[key] = r - b.byDir, b.mods = nil, nil } // Delete takes a key out. func (b *Builds) Delete(key string) { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.records[key]; ok { - delete(b.records, key) - b.byDir, b.mods = nil, nil - } -} - -// dirKey names a directory of a repository whichever way the repository is spelled: a build record names it by its clone -// URL, a forge message by "owner/name" (issue 1281). -func dirKey(repository, dir string) string { - return strings.ToLower(delivery.RepositoryName(repository)) + "\x00" + strings.Trim(dir, "/") -} - -// Module is the module a build record of the repository names the directory as its path, if one does. -func (b *Builds) Module(repository, dir string) (string, bool) { - b.mu.Lock() - defer b.mu.Unlock() - if b.byDir == nil { - b.byDir = map[string]delivery.BuildRecord{} - for _, r := range b.records { - k := dirKey(r.Repository, r.Path) - if old, ok := b.byDir[k]; !ok || r.At.After(old.At) || (r.At.Equal(old.At) && r.Module < old.Module) { - b.byDir[k] = r - } - } - } - r, ok := b.byDir[dirKey(repository, dir)] - return r.Module, ok -} - -// ModuleOf is the module of a changed directory: the one a build record of the repository names it the path of; a -// directory no build record names is the directory's last element (the catalogue's rule for nearly every manifest). -func (b *Builds) ModuleOf(repository, dir string) string { - if m, ok := b.Module(repository, dir); ok { - return m - } - return path.Base(strings.Trim(dir, "/")) -} - -// Known is a module the builds know of in a repository: its directory there and its newest successful trunk build -// (delivery.BuildRecord.IsTrunkBuild) if any. The newest key of a module is any state and ref, so it is not what -// finds this (mesh-sdk go/delivery/buildrecord.go NewestKey); the records are. -type Known struct { - Module string - Repository string - Path string - Trunk *delivery.BuildRecord + delete(b.records, key) } func newer(a, b delivery.BuildRecord) bool { return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID) } -// Modules are the modules built from a repository, by name. A module whose newest successful trunk build is of the -// repository belongs to it with that record's path; a module with no such build belongs to the repository of its newest -// record of any kind. -func (b *Builds) Modules(repository string) []Known { - b.mu.Lock() - defer b.mu.Unlock() - if b.mods == nil { - b.mods = map[string]*moduleBuilds{} - for _, r := range b.records { - m := b.mods[r.Module] - if m == nil { - m = &moduleBuilds{any: r} - b.mods[r.Module] = m - } else if newer(r, m.any) { - m.any = r - } - if r.IsTrunkBuild() && (m.trunk == nil || newer(r, *m.trunk)) { - c := r - m.trunk = &c - } - } - } - var out []Known - for name, m := range b.mods { - k := Known{Module: name, Repository: m.any.Repository, Path: strings.Trim(m.any.Path, "/"), Trunk: m.trunk} - if m.trunk != nil { - k.Repository, k.Path = m.trunk.Repository, strings.Trim(m.trunk.Path, "/") - } - if delivery.SameRepository(k.Repository, repository) { - out = append(out, k) - } - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - // TrunkOf is the newest successful trunk build record of a module, in whatever repository it was built from; nil when // there is none (issue 1727). func (b *Builds) TrunkOf(module string) *delivery.BuildRecord { @@ -153,3 +64,27 @@ func (b *Builds) TrunkOf(module string) *delivery.BuildRecord { } return best } + +// PathOf is the directory of a module of the repository as its newest build record of that repository names it, its +// newest successful trunk build's first ("." for the repository's root), and false when no record of the repository +// names the module (issue 1614). +func (b *Builds) PathOf(module, repository string) (string, bool) { + var best *delivery.BuildRecord + trunk := false + b.mu.Lock() + defer b.mu.Unlock() + for _, r := range b.records { + if r.Module != module || !delivery.SameRepository(r.Repository, repository) { + continue + } + t := r.IsTrunkBuild() + if best == nil || (t && !trunk) || (t == trunk && newer(r, *best)) { + c := r + best, trunk = &c, t + } + } + if best == nil { + return "", false + } + return pathOf(strings.Trim(best.Path, "/")), true +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go index bdadf94..a43f4e9 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -3,6 +3,7 @@ package index import ( "context" "errors" + "reflect" "strings" "testing" @@ -43,7 +44,7 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t } c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"} - if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want { + if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || !reflect.DeepEqual(c.Pulls[0].Fps["avahi"], want) { t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want) } if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) { @@ -55,7 +56,7 @@ func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t if err := ix.Apply(context.Background(), again); err != nil { t.Fatal(err) } - if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet { + if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.FPSet != c.FPSet { t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet) } } @@ -73,8 +74,8 @@ func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/ func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3 avahi := catalogueManifest(t, "avahi") - // The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory. - other := catalogueManifest(t, "ca-trust") + // The second pull request carries avahi's manifest changed (a line added at its end) for the same directory. + other := avahi + "\n" ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 2f1bf70..89a1ac9 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "slices" "strings" "sync" @@ -46,7 +47,12 @@ type Indexer struct { // Manifests is the manifests bucket; nil writes no manifests (issue 1449). Manifests Store Builds *Builds - Logf func(format string, args ...any) + // Named asks the forge for the git object of each named path of the module directories given, at a commit + // (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a + // manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths + // has none. + Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) + Logf func(format string, args ...any) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow @@ -78,11 +84,15 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } } var digests map[string]string + var held map[string]delivery.Module if f.Kind == delivery.ForgePull { var err error if digests, err = ix.keepManifests(ctx, f); err != nil { return err } + if held, err = ix.heldModules(ctx); err != nil { + return err + } } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) @@ -101,7 +111,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } cur = &c } - next := ix.next(cur, f, digests) + next := ix.next(cur, f, digests, held) if next != nil { ix.withPreviews(ctx, next) if err := ix.refuseDisagreement(*next); err != nil { @@ -207,8 +217,8 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, // A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of // its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a // pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by -// directory. -func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate { +// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir). +func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate { var pulls []delivery.CandidatePull var base uint64 if cur != nil { @@ -225,7 +235,7 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s switch { case f.DirsKnown: for _, dir := range f.Dirs { - m := ix.Builds.ModuleOf(f.Repository, dir) + m := moduleOfDir(held, f, dir) if !slices.Contains(entry.Modules, m) { entry.Modules = append(entry.Modules, m) } @@ -280,7 +290,7 @@ func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error { first[module] = p continue } - if before.Fps[module] != m { + if !reflect.DeepEqual(before.Fps[module], m) { err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written", ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest) return err diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index 8fa182b..74bcc57 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -148,27 +148,35 @@ func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the gr } } -func TestModuleOfADirectory(t *testing.T) { // 1238/AC3 +func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the module of a directory is the indexer's own (I2, ADR 0501 clause 6) ix, st, _ := newIndexer(t) - // modules/odd is named by a build record of the repository (module odd-name); modules/plain is named by none. - if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { + mods := newMem() + ix.Modules = mods + ctx := context.Background() + // The trunk holds modules/odd as the module odd-name (its manifest names it so); modules/plain is held by none. + merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/odd": "t"}) + merged.Manifests["modules/odd"] = `{"module":"odd-name","version":"1"}` + if err := ix.Apply(ctx, merged); err != nil { + t.Fatal(err) + } + if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { t.Fatal(err) } c, _ := read(t, st, "feat/x") if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" { - t.Errorf("modules = %v, want the record's module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) + t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) } - // A record of another repository does not name this repository's directory. - if m, ok := ix.Builds.Module("novox/other", "modules/a"); ok { - t.Errorf("modules/a of novox/other is %q", m) + held := map[string]delivery.Module{"x": {Repository: "novox/other", Path: "modules/a"}} + if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok { + t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name) } - // The newest record of a directory wins. - ix.Builds.Put(delivery.BuildKey("renamed", "build-9"), buildRecord(t, "novox/mesh-delivery", "renamed", "modules/a", "2026-10-19T10:00:00Z")) - if m := ix.Builds.ModuleOf("novox/mesh-delivery", "modules/a"); m != "renamed" { - t.Errorf("modules/a = %q, want the newest record's module", m) + // A manifest the pull message carries names its module, whatever directory it is in. + f := pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/odd") + f.Manifests = map[string]string{"modules/odd": `{"module":"renamed","version":"1"}`} + if name := moduleOfDir(held, f, "modules/odd"); name != "renamed" { + t.Errorf("modules/odd = %q, want the module its manifest names", name) } } - func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4 ix, st, _ := newIndexer(t) ctx := context.Background() diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index 250b78b..3af599e 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -37,6 +37,8 @@ type moduleInputs struct { Languages map[string]string // None2 is why the module has no fingerprint of the new form. None2 string + // ArtifactPaths are the paths each artifact naming them names, as its manifest names them (delivery.Module.ArtifactPaths). + ArtifactPaths map[string][]string } // baseFP answers a base module's new-form fingerprint, or why it has none. @@ -55,17 +57,55 @@ type baseFP func(module string) (fp, none string) // here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve // nothing and have no toolchain. A base is a build.on entry that names a module. func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs { - m, err := catalogue.ParseManifest([]byte(raw)) + return inputsOfNamed(module, raw, tree, locks, nil, base) +} + +// inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery +// Forge.Paths, PreviewModule.Paths). A module whose every artifact names its paths takes its tree input from them and its +// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs +// gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it +// none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given. +func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, base baseFP) moduleInputs { + m, err := catalogue.ParseManifestForInputs([]byte(raw)) if err != nil { return moduleInputs{None2: "the manifest cannot be read: " + err.Error()} } if m.Module != module { return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)} } - if tree == "" { - return moduleInputs{None2: "the merge names no tree for the module's directory"} + var artifactPaths map[string][]string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if len(a.Paths) > 0 { + if artifactPaths == nil { + artifactPaths = map[string][]string{} + } + artifactPaths[a.Name] = slices.Clone(a.Paths) + } + } } - kept := delivery.KeptInputs2{Tree: tree} + var named map[string]string + if paths, every := m.NamedPaths(); every { + var missing []string + named = map[string]string{} + for _, p := range paths { + if objects[p] == "" { + missing = append(missing, p) + continue + } + named[p] = objects[p] + } + if len(missing) > 0 { + return moduleInputs{None2: "the message carries no git object of the named paths " + strings.Join(missing, ", "), ArtifactPaths: artifactPaths} + } + if tree, err = delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), named); err != nil { + return moduleInputs{None2: err.Error(), ArtifactPaths: artifactPaths} + } + } + if tree == "" { + return moduleInputs{None2: "the merge names no tree for the module's directory", ArtifactPaths: artifactPaths} + } + kept := delivery.KeptInputs2{Tree: tree, Paths: named} contexts := false var missing []string langs := languagesOf(m) @@ -94,7 +134,7 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo } sort.Strings(kept.Bases) sort.Strings(missing) - out := moduleInputs{Kept: &kept} + out := moduleInputs{Kept: &kept, ArtifactPaths: artifactPaths} if len(langs) > 0 { out.Languages = langs } diff --git a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go index 4794ab6..7aacda2 100644 --- a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go @@ -1,7 +1,6 @@ package index import ( - "context" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -51,49 +50,17 @@ func liveIndexer(t *testing.T) (*Indexer, *memStore) { return ix, mods } -func TestMergeWritesEveryModuleTheLiveRecordsNameInTheRepository(t *testing.T) { // 1281/AC3 - ix, mods := liveIndexer(t) - msg := mergedMsg("novox/mesh-catalog", map[string]string{ - "modules/zsh": liveTree, "modules/mesh-delivery": "newtree", "modules/mailu": "newtree", "modules/sonarr": "newtree", - }) - if err := ix.Apply(context.Background(), msg); err != nil { - t.Fatal(err) - } - for _, name := range []string{"zsh", "mesh-delivery", "mailu"} { - m, ok := moduleOf(t, mods, name) - if !ok { - t.Errorf("%s: no module key written", name) - continue - } - if (m.FP == "") == (m.None == "") || m.Repository != "novox/mesh-catalog" || m.Commit != merge { - t.Errorf("%s: module = %+v, want an fp or a none, repository novox/mesh-catalog, commit %s", name, m, merge) - } - } - // the tree of zsh at the merge is the tree its inputs record, so its fingerprint is the one those inputs give - var in delivery.SourceInputs - in.Module, in.Tree = "zsh", liveTree - in.Bases = []string{"sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"} - in.Toolchains = map[string]string{"code": "typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"} - want, _ := delivery.SourceFingerprint(in) - if m, _ := moduleOf(t, mods, "zsh"); m.FP != want { - t.Errorf("zsh fp = %q, want %q", m.FP, want) - } - if _, ok := mods.vals["sonarr"]; ok { - t.Error("the key of a module of mesh-media-catalog was written at a merge of mesh-catalog") - } -} - -func TestChangedDirectoryMapsToTheModuleItsLiveRecordNamesByPath(t *testing.T) { // 1281/AC4 +func TestTheLiveRecordsGiveTheDirectoryOfAModuleOfTheirRepository(t *testing.T) { // 1614 (I3: the records name directories, never list modules) ix, _ := liveIndexer(t) - for dir, want := range map[string]string{"modules/zsh": "zsh", "modules/mesh-delivery": "mesh-delivery", "modules/mailu/": "mailu"} { - if got, ok := ix.Builds.Module("novox/mesh-catalog", dir); !ok || got != want { - t.Errorf("Module(novox/mesh-catalog, %q) = %q, %v; want %q", dir, got, ok, want) + for module, want := range map[string]string{"zsh": "modules/zsh", "mesh-delivery": "modules/mesh-delivery", "mailu": "modules/mailu"} { + if got, ok := ix.Builds.PathOf(module, "novox/mesh-catalog"); !ok || got != want { + t.Errorf("PathOf(%s, novox/mesh-catalog) = %q, %v; want %q", module, got, ok, want) } } - if got, ok := ix.Builds.Module("novox/mesh-catalog", "modules/sonarr"); ok { - t.Errorf("Module found %q for a directory of mesh-media-catalog under mesh-catalog", got) + if got, ok := ix.Builds.PathOf("sonarr", "novox/mesh-catalog"); ok { + t.Errorf("PathOf found %q for a module of mesh-media-catalog under mesh-catalog", got) } - if got := ix.Builds.ModuleOf("novox/mesh-media-catalog", "modules/sonarr"); got != "sonarr" { - t.Errorf("ModuleOf(mesh-media-catalog, modules/sonarr) = %q, want sonarr", got) + if got, ok := ix.Builds.PathOf("sonarr", "novox/mesh-media-catalog"); !ok || got != "modules/sonarr" { + t.Errorf("PathOf(sonarr, mesh-media-catalog) = %q, %v; want modules/sonarr", got, ok) } } diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go index d1883b1..740e9ce 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -2,27 +2,12 @@ package index import ( "context" - "encoding/json" "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests -// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the -// module a build record names the directory the path of. -func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string { - var m struct { - Module string `json:"module"` - } - if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" { - return m.Module - } - name, _ := ix.Builds.Module(repository, dir) - return name -} - // keepManifest writes manifests. for a manifest as the forge answered it, create-once (issue 1449): a key that // exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for // one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the @@ -66,7 +51,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) { digests := make(map[string]string, len(f.Manifests)) for dir, raw := range f.Manifests { - module := ix.moduleNameOf(f.Repository, dir, raw) + module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module digest, err := ix.keepManifest(ctx, module, raw) switch { case err == nil: diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 8c26287..08455bb 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -96,7 +96,8 @@ func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a if len(manifests.vals) != 0 || len(logs) == 0 { t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs) } - if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" { - t.Errorf("module = %+v (%v), want no manifest digest", mod, ok) + // A manifest that is not JSON names no module (I1): nothing is written for its directory. + if mod, ok := moduleOf(t, mods, "avahi"); ok { + t.Errorf("module = %+v, want none written for a manifest naming no module", mod) } } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 7e55e43..7749238 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -2,79 +2,71 @@ package index import ( "context" - "sort" - "strings" + "maps" + "slices" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// The reasons a module has no fingerprint (ADR 0437 §5; issue 1209's decision). -const ( - // NoneNarrow is for a module whose image compiles a Go program from a context: the context is the hash of the build - // source the builder handed it, not a git tree. - NoneNarrow = "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4" - // NoneContext is for a context that is a repository's root tree. - NoneContext = "an artifact reads the root tree of a repository the indexer does not know the tree of before release 4" - // NoneNoBuild is for a module no successful trunk build recorded the inputs of. - NoneNoBuild = "no successful trunk build recorded its inputs" -) - -// fingerprintOf is the module's fingerprint at the merge, or none and why: its directory's tree from the merged -// message and the bases, contexts and toolchains its newest successful trunk build recorded. -func fingerprintOf(k Known, trees map[string]string) (fp, none string) { - if k.Trunk == nil { - return "", NoneNoBuild - } - in := *k.Trunk.SourceInputs - artifacts := make([]string, 0, len(in.Contexts)) - for a := range in.Contexts { - artifacts = append(artifacts, a) - } - sort.Strings(artifacts) - for _, a := range artifacts { - if strings.HasPrefix(in.Contexts[a], "narrow:") { - return "", NoneNarrow - } - } - if len(artifacts) > 0 { - return "", NoneContext - } - in.Tree = trees[k.Path] // the directory "" is the repository's root - return delivery.SourceFingerprint(in) -} - -// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and -// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it -// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already -// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it -// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449, -// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build -// module reports its toolchain. +// applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that +// holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest, +// its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as +// the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is +// computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain. +// +// The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue +// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's +// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the +// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420 +// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other. +// A value the module already holds is not written again. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { digests, err := ix.keepManifests(ctx, f) if err != nil { return err } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + objects, namedErr := ix.namedObjects(ctx, f) overlay := map[string]delivery.Module{} - for _, k := range ix.Builds.Modules(f.Repository) { - key, err := delivery.ModulesKey(k.Module) - if err != nil { - ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, k.Module, err) + dirOf := map[string]string{} + for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) { + raw := f.Manifests[dir] + name := manifestModule(raw) + if name == "" { + ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir) continue } - fp, none := fingerprintOf(k, f.Trees) - next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path], Path: pathOf(k.Path)} - // The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which - // names what is missing, from its manifest and lock hashes (inputs.go). - raw, has := f.Manifests[k.Path] - switch { - case !has: - next.None2 = None2NoManifest - case next.Manifest == "": + key, err := delivery.ModulesKey(name) + if err != nil { + ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err) + continue + } + cur, has := held[key] + if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) { + ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository) + continue + } + if other, dup := dirOf[name]; dup { + if !has || cur.Path != pathOf(dir) { + ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other) + continue + } + ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir) + } + dirOf[name] = dir + next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)} + if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" - default: - in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx)) - next.Inputs2, next.None2 = in.Kept, in.None2 + } else { + in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx)) + next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths + if in.Kept == nil && namedErr[dir] != "" { + next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir] + } } overlay[key] = next } @@ -118,3 +110,42 @@ func pathOf(dir string) string { } func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m } + +// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact +// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call +// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them. +func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) { + objects := map[string]map[string]string{} + need := map[string][]string{} + for dir, raw := range f.Manifests { + m, err := catalogue.ParseManifestForInputs([]byte(raw)) + if err != nil { + continue // inputsOfNamed says why + } + paths, every := m.NamedPaths() + if !every { + continue + } + if carried := f.Paths[dir]; len(carried) > 0 { + objects[dir] = carried + continue + } + need[dir] = paths + } + if len(need) == 0 || ix.Named == nil { + return objects, nil + } + got, err := ix.Named(ctx, f.Repository, f.Merge, need) + if err != nil { + words := map[string]string{} + for dir := range need { + words[dir] = err.Error() + } + ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err) + return objects, words + } + for dir := range need { + objects[dir] = got[dir] + } + return objects, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 36eee4f..ca0cf36 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -3,6 +3,7 @@ package index import ( "context" "encoding/json" + "path" "strings" "testing" @@ -51,8 +52,22 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2 return [2]any{k, v} } +// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its +// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that +// holds a module.json). func mergedMsg(repo string, trees map[string]string) delivery.Forge { - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees} + manifests := map[string]string{} + for dir := range trees { + manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}` + } + return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} +} + +func nameOfDir(dir string) string { + if dir == "" { + return "r" + } + return path.Base(dir) } func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) { @@ -70,126 +85,6 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) { const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` -func TestModuleFingerprintFromTheMergeTreeAndTheRecordedInputs(t *testing.T) { // 1239/AC1 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "modules/b": "other"})); err != nil { - t.Fatal(err) - } - want, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if none != "" { - t.Fatal(none) - } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != want || m.None != "" || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Src != 0 { - t.Errorf("module = %+v (%v), want fp %s", m, ok, want) - } - if _, ok := mods.vals["b"]; ok { - t.Error("a key was written for a directory no build names") - } -} - -func TestMergeOfTheRecordedTreeReproducesTheRecordedFingerprint(t *testing.T) { // 1239/AC2 - // mailu, build-1792391134620131370: mesh-catalog at b4527d9091e972f28cb365623d9280045e7741a3, dir modules/mailu. Its - // inputs and recorded fingerprint are the live ones mesh-sdk go/delivery/fingerprint_test.go TestFingerprintLiveBuilds - // captured (its tree is `git rev-parse :modules/mailu`, its bases from the build's [bases] log line). - const tree = "cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e" - const recorded = "src1:539f2e64000d436c572ef53b40e524012455945f8c5a1c6bb173ee0707e88ef4" - inputs := `{"Module":"mailu","Tree":"` + tree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` + - `"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "", "2026-10-19T09:00:00Z", inputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-catalog", map[string]string{"modules/mailu": tree})); err != nil { - t.Fatal(err) - } - if m, ok := moduleOf(t, mods, "mailu"); !ok || m.FP != recorded { - t.Errorf("module = %+v (%v), want fp %s", m, ok, recorded) - } -} - -func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1 - // A Go program's build source as the builder records it (build-agent's record, mesh-sdk go/delivery/testdata): - // the context "server" is "narrow:" and a hash. - inputs := `{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", inputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-controller", map[string]string{"modules/build-agent": "newtree"})); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "build-agent") - if !ok || m.FP != "" || !strings.Contains(m.None, "Go program") || !strings.Contains(m.None, "release 4") { - t.Errorf("module = %+v (%v), want none naming the Go program's build source and release 4", m, ok) - } -} - -func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4 - inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":"the build resolves packages when built"}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "the build resolves packages when built" { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5 - inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":{"web":"f00dcafef00dcafef00dcafef00dcafef00dcafe"},"Toolchains":null,"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != "" || !strings.Contains(m.None, "root tree") || !strings.Contains(m.None, "release 4") { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/other": "t"})) - if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "no tree" { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7 - ix, mods := modulesIndexer(t, - rec(t, "build-1", "failed-only", "novox/mesh-delivery", "modules/f", "main", "failed", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "", "2026-10-19T09:00:00Z", ""), - ) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/f": "t", "modules/g": "t", "modules/h": "t"})) - for _, name := range []string{"failed-only", "branch-only", "no-inputs"} { - if m, ok := moduleOf(t, mods, name); !ok || m.FP != "" || !strings.Contains(m.None, "no successful trunk build") { - t.Errorf("%s: module = %+v (%v)", name, m, ok) - } - } -} - -func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest key is only a hint) - older := strings.Replace(aInputs, `"Bases":["registry.example/base@sha256:aaa"]`, `"Bases":["registry.example/base@sha256:old"]`, 1) - ix, mods := modulesIndexer(t, - rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", older), - rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T10:00:00Z", aInputs), - rec(t, "build-3", "a", "novox/mesh-delivery", "modules/a", "main", "failed", "2026-10-19T11:00:00Z", older), - rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "", "2026-10-19T12:00:00Z", older), - ) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if m, _ := moduleOf(t, mods, "a"); m.FP != want { - t.Errorf("fp = %s, want the 10:00 trunk build's %s", m.FP, want) - } -} - -func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8 - ix, mods := modulesIndexer(t, - rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1))) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/c": "t"})) - if _, ok := mods.vals["c"]; ok { - t.Error("the key of a module of another repository was written") - } - if _, ok := mods.vals["a"]; !ok { - t.Error("the key of the merged repository's module was not written") - } -} - func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing) ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}) @@ -211,21 +106,6 @@ func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 } } -func TestARecordAsTheControllerWritesItGivesAFingerprint(t *testing.T) { // 1281/AC8 - // The layout of mesh-controller internal/chain/records.go Build (commit 1f708e2d8d72): an outcome has no state key, a build - // asked by a plan after a merge no ref, and the repository is the clone URL. - raw := `{"id":"build-1","repository":"http://novox.internal:20000/novox/mesh-delivery.git","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20",` + - `"path":"modules/a","source-fingerprint":"src1:ignored","source-inputs":` + aInputs + `,"at":"2026-10-19T09:00:00Z"}` - ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("a", "build-1"), []byte(raw)}) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP == "" || m.None != "" { - t.Errorf("module = %+v (%v), want a fingerprint, not none", m, ok) - } -} - func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2 ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), @@ -263,3 +143,75 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means t.Errorf("module = %+v (%v), want path %q", m, ok, delivery.RootPath) } } + +func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 + ix, mods, _ := manifestsIndexer(t) // no build record of any module + avahi := catalogueManifest(t, "avahi") + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "avahi") + if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" { + t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok) + } + if m.FP != "" || m.None != "" { + t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m) + } + if m.FP2 == "" && m.None2 == "" { + t.Errorf("module = %+v, want a new-form fingerprint or its reason", m) + } +} + +func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1 + ix, mods := modulesIndexer(t, rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/b": "t"}) + f.Trees["modules/c"] = "t" // a directory whose manifest the message does not carry + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + for name, want := range map[string]bool{"a": true, "b": true, "c": false} { + if _, ok := mods.vals[name]; ok != want { + t.Errorf("modules.%s written %v, want %v", name, ok, want) + } + } +} + +func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7) + ix, mods := modulesIndexer(t) + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + ctx := context.Background() + if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil { + t.Fatal(err) + } + before := string(mods.vals["a"]) + if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil { + t.Fatal(err) + } + if string(mods.vals["a"]) != before || len(logs) == 0 { + t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs) + } +} + +func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { // I1 + ix, mods := modulesIndexer(t) + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "other/a": "t2"}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, ok := moduleOf(t, mods, "a"); !ok || m.Path != "modules/a" || len(logs) == 0 { + t.Errorf("module = %+v (%v), logs %v: want the first directory by name, and the other said", m, ok, logs) + } + // Once held, the directory the bucket holds stays the module's. + f = mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "aaa/a": "t3"}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" { + t.Errorf("module = %+v, want the held directory kept", m) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/named_test.go b/modules/mesh-delivery-indexer/internal/index/named_test.go new file mode 100644 index 0000000..13f99c5 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/named_test.go @@ -0,0 +1,133 @@ +package index + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0499 clauses 2 and 3 (I7): a module whose every artifact names its paths takes its tree input from them, by mesh-sdk's +// one function, and keeps their objects, so a computation from kept inputs gives the merge's value. The manifest is +// mesh-sdk's module.json at 9cb4d59 (its lib names src, test, package.json, package-lock.json and tsconfig.json). + +var sdkObjects = map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", + "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"} + +func TestAModuleNamingItsPathsTakesItsTreeFromThemAndKeepsTheirObjects(t *testing.T) { // I7, ADR 0499 clause 2 + ix, mods, _ := manifestsIndexer(t) + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, map[string]map[string]string{"": {"package-lock.json": avahiGoSum}}) + f.Paths = map[string]map[string]string{"": sdkObjects} + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "mesh-sdk") + if !ok || m.Inputs2 == nil { + t.Fatalf("module = %+v (%v)", m, ok) + } + want, err := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects) + if err != nil { + t.Fatal(err) + } + if m.Inputs2.Tree != want || !reflect.DeepEqual(m.Inputs2.Paths, sdkObjects) { + t.Errorf("inputs = %+v, want the named tree %s and the objects kept", m.Inputs2, want) + } + if got := m.ArtifactPaths["lib"]; strings.Join(got, ",") != "src,test,package.json,package-lock.json,tsconfig.json" { + t.Errorf("artifact paths = %v, want lib's paths as its manifest names them", m.ArtifactPaths) + } + // Recomputed from what is kept, with no merge, the module is the same (ADR 0499 clause 2). + before := string(mods.vals["mesh-sdk"]) + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if string(mods.vals["mesh-sdk"]) != before { + t.Errorf("a recompute changed the module: %s, was %s", mods.vals["mesh-sdk"], before) + } +} + +func TestAMessageWithoutTheNamedPathsObjectsGivesTheModuleNoneNamingThem(t *testing.T) { // I7, ADR 0499 clause 3 + ix, mods, _ := manifestsIndexer(t) + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "mesh-sdk") + if m.FP2 != "" || !strings.Contains(m.None2, "named paths") || !strings.Contains(m.None2, "tsconfig.json") { + t.Errorf("module = %+v, want none naming the paths the message did not carry", m) + } +} + +func TestAModuleNamingNoPathsKeepsItsDirectorysTree(t *testing.T) { // I7, ADR 0499 clause 2: every other module is unchanged + in := inputsOfNamed("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, map[string]string{"x": "1111111111111111111111111111111111111111"}, noBases) + if in.Kept == nil || in.Kept.Tree != "avahitree" || in.Kept.Paths != nil { + t.Errorf("inputs = %+v, want the directory's tree and no paths", in.Kept) + } +} + +func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) { // I8, ADR 0499 clause 2 (the merge preview) + ix, _, _ := manifestsIndexer(t) + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + digest, err := ix.keepManifest(context.Background(), "mesh-sdk", raw) + if err != nil { + t.Fatal(err) + } + named, _ := delivery.NamedTree(digest, sdkObjects) + with := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}} + without := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: "roottree", Manifest: digest}}} + if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without); !strings.Contains(got.None2, "named paths") { + t.Errorf("without the objects: %+v, want none naming the named paths", got) + } + got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with) + if strings.Contains(got.None2, "named paths") || got.Manifest != digest { + t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got) + } + if got.FP2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) { + t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got) + } +} + +func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths + ix, mods, _ := manifestsIndexer(t) + type ask struct { + repository, sha string + paths map[string][]string + } + var asked []ask + ix.Named = func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + asked = append(asked, ask{repository, sha, paths}) + return map[string]map[string]string{"": sdkObjects}, nil + } + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree", "modules/a": "t"}), map[string]string{"": raw, "modules/a": `{"module":"a","version":"1"}`}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if len(asked) != 1 || asked[0].repository != "novox/mesh-sdk" || asked[0].sha != merge || len(asked[0].paths) != 1 || + strings.Join(asked[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" { + t.Fatalf("asked %+v, want the root's named paths of the merge commit, once, and nothing for a module naming none", asked) + } + m, _ := moduleOf(t, mods, "mesh-sdk") + want, _ := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects) + if m.Inputs2 == nil || m.Inputs2.Tree != want { + t.Errorf("module = %+v, want the named tree %s", m, want) + } +} + +func TestAForgeThatCannotSayTheObjectsGivesTheModuleNoneWithItsWords(t *testing.T) { // I6 + ix, mods, _ := manifestsIndexer(t) + ix.Named = func(context.Context, string, string, map[string][]string) (map[string]map[string]string, error) { + return nil, errors.New("gitea: the path tsconfig.json of \"\" is not at the commit") + } + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "mesh-sdk"); m.FP2 != "" || !strings.Contains(m.None2, "is not at the commit") { + t.Errorf("module = %+v, want none with the forge's words", m) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/newform_test.go b/modules/mesh-delivery-indexer/internal/index/newform_test.go index a610a9c..31247a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/newform_test.go +++ b/modules/mesh-delivery-indexer/internal/index/newform_test.go @@ -7,17 +7,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go). -func wantAvahiFP(t *testing.T) string { - t.Helper() - fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if none != "" { - t.Fatal(none) - } - return fp -} - func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1 avahi := catalogueManifest(t, "avahi") ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", @@ -31,16 +20,14 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t if !ok { t.Fatal("no module avahi") } - todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum { t.Errorf("module = %+v, want its manifest digest and inputs", m) } if m.None2 != "toolchain not yet reported" || m.FP2 != "" { t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2) } - if m.FP != todays || m.None != "" { - t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays) + if m.FP != "" || m.None != "" { + t.Errorf("fp = %q none = %q, want no value of today's form (I4)", m.FP, m.None) } } @@ -55,17 +42,15 @@ func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) } } -func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3 +func TestAMergedMessageWithoutManifestsWritesNoModule(t *testing.T) { // 1444/AC3 as R4e has it: the modules are the manifests' (I1) ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}) + f.Manifests = nil + if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" { - t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok) - } - if len(manifests.vals) != 0 { - t.Errorf("manifests holds %d keys", len(manifests.vals)) + if len(mods.vals) != 0 || len(manifests.vals) != 0 { + t.Errorf("modules %d keys, manifests %d keys, want none: a message naming no manifest names no module", len(mods.vals), len(manifests.vals)) } } diff --git a/modules/mesh-delivery-indexer/internal/index/own.go b/modules/mesh-delivery-indexer/internal/index/own.go new file mode 100644 index 0000000..bcb0f81 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/own.go @@ -0,0 +1,81 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "path" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// From R4e the indexer lists a repository's modules from what it holds itself (novox/hq ADR 0501 clause 6, issue 1851): +// a merged message's manifests name the modules of the merge, and the modules bucket names the modules it wrote before, +// with their repository and directory. The controller's build records name none of them (ADR 0453 clause 5 keeps only +// the builder's own input from them until R4g, builds.go). + +// manifestModule is the module a manifest names itself (its `module`), or "" when it names none or is not JSON. Only the +// name is read here; the manifest is read whole by the catalogue's parser (inputs.go). +func manifestModule(raw string) string { + var m struct { + Module string `json:"module"` + } + if json.Unmarshal([]byte(raw), &m) != nil { + return "" + } + return m.Module +} + +// heldModules are the values the modules bucket holds, by key (the module's name). A value that is not a module is left +// out and said. +func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module, error) { + out := map[string]delivery.Module{} + if ix.Modules == nil { + return out, nil + } + keys, err := ix.Modules.Keys(ctx) + if err != nil { + return nil, fmt.Errorf("listing the modules: %w", err) + } + for _, key := range keys { + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("modules.%s: %w", key, err) + } + if !found { + continue + } + m, err := delivery.DecodeModule(value) + if err != nil { + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + continue + } + out[key] = m + } + return out, nil +} + +// moduleAt is the module whose directory in the repository is dir, as the modules bucket holds it. +func moduleAt(held map[string]delivery.Module, repository, dir string) (string, bool) { + want := pathOf(strings.Trim(dir, "/")) + for name, m := range held { + if m.Path == want && delivery.SameRepository(m.Repository, repository) { + return name, true + } + } + return "", false +} + +// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else +// the one the modules bucket holds at that directory of the repository, else the directory's last element (the +// catalogue's rule for nearly every manifest). +func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string { + if name := manifestModule(f.Manifests[dir]); name != "" { + return name + } + if name, ok := moduleAt(held, f.Repository, dir); ok { + return name + } + return path.Base(strings.Trim(dir, "/")) +} diff --git a/modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go b/modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go new file mode 100644 index 0000000..48ee007 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/paths_backfill_test.go @@ -0,0 +1,67 @@ +package index + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Issue 1614: a modules key written before the indexer learned paths (mesh-tools and node-tools live) gets its directory +// from the module's newest build record without waiting for a merge of its repository, once, changing no fingerprint. The +// read of the controller's build records for it goes in R4g with build-agent (the lead's decision, 2026-10-22). + +func heldValue(t *testing.T, st *memStore, name, path string) { + t.Helper() + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-tools", Commit: "e0fadc43d9dda0bc24bd19bb05b021dd87da8582", + FP2: "src2:" + hex64('2'), Manifest: hex64('9'), Path: path}) + if err != nil { + t.Fatal(err) + } + st.put(name, data) +} + +func hex64(c byte) string { + b := make([]byte, 64) + for i := range b { + b[i] = c + } + return string(b) +} + +func TestAnEmptyPathIsFilledOnceFromTheModulesNewestBuildRecord(t *testing.T) { // 1614 + ix, mods := modulesIndexer(t, + [2]any{delivery.BuildKey("mesh-tools", "build-1"), buildRecord(t, "novox/mesh-tools", "mesh-tools", "", "2026-10-19T09:00:00Z")}, + [2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "node-tools", "2026-10-19T09:00:00Z")}) + heldValue(t, mods, "mesh-tools", "") + heldValue(t, mods, "node-tools", "") + before, _ := moduleOf(t, mods, "node-tools") + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"mesh-tools": delivery.RootPath, "node-tools": "node-tools"} { + m, _ := moduleOf(t, mods, name) + if m.Path != want || m.FP2 != before.FP2 || m.Commit != before.Commit { + t.Errorf("%s = %+v, want path %q and nothing else changed", name, m, want) + } + } + writes := mods.writes + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if mods.writes != writes { + t.Errorf("the paths were written again (%d writes, was %d): once only", mods.writes, writes) + } +} + +func TestASetPathIsNeverOverwrittenByABuildRecord(t *testing.T) { // 1614 + ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("node-tools", "build-2"), buildRecord(t, "novox/mesh-tools", "node-tools", "elsewhere", "2026-10-19T09:00:00Z")}) + heldValue(t, mods, "node-tools", "node-tools") + writes := mods.writes + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "node-tools"); m.Path != "node-tools" || mods.writes != writes { + t.Errorf("module = %+v after %d writes (was %d), want its path kept and nothing written", m, mods.writes, writes) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/predefined_test.go b/modules/mesh-delivery-indexer/internal/index/predefined_test.go new file mode 100644 index 0000000..d83f31f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/predefined_test.go @@ -0,0 +1,29 @@ +package index + +import ( + "strings" + "testing" +) + +// Issue 1751: a manifest whose config value falls back to a predefined key (novox/hq ADR 0348) is read by the indexer, so +// the module gets its new-form inputs. The manifests are claude-code's, dunst's and xterm's module.json from mesh-catalog +// at 835bf22b. +func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) { // 1751 + for _, module := range []string{"claude-code", "dunst", "xterm"} { + raw := fixture(t, "catalogue-835bf22b-"+module) + if !strings.Contains(raw, `"predefined"`) { + t.Fatalf("%s: the fixture names no predefined key", module) + } + in := inputsOf(module, raw, "newtree", nil, func(string) (string, string) { return "src2:" + hex64('1'), "" }) + if in.Kept == nil || strings.Contains(in.None2, "cannot be read") { + t.Errorf("%s: inputs = %+v, want the manifest read and its inputs kept", module, in) + } + } +} + +func TestAPredefinedKeyOfAnotherTypeThanItsValueIsStillRefused(t *testing.T) { // 1751: the read judges what it can + raw := strings.Replace(fixture(t, "catalogue-835bf22b-xterm"), `"predefined": "font-size"`, `"predefined": "font-size", "trusted": "yes"`, 1) + if in := inputsOf("xterm", raw, "newtree", nil, func(string) (string, string) { return "", "none" }); in.Kept != nil { + t.Errorf("inputs = %+v, want a manifest the catalogue refuses still refused", in) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index ddd0a51..5e8b454 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "maps" + "reflect" "slices" "strings" @@ -26,7 +27,7 @@ const ( NonePreviewNoModule = "the merge preview names no manifest for the module's directory" // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. NonePreviewNoManifest = "the manifest of the merge is not kept" - // NoneNoDirectory is for a module whose directory in its repository the builds do not tell. + // NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells. NoneNoDirectory = "the module's directory in its repository is not known" ) @@ -143,9 +144,27 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool if !found && !stale { continue } + // A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory + // whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6). dirs := map[string]string{} - for _, k := range ix.Builds.Modules(p.Repository) { - dirs[k.Module] = k.Path + if held, err := ix.heldModules(ctx); err == nil { + for name, m := range held { + if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" { + dirs[name] = m.Path + } + } + } else { + ix.logf("the modules cannot be listed, so no touched module's directory is known: %v", err) + } + for _, m := range p.Modules { + if _, known := dirs[m]; known || p.Fps[m].Manifest == "" { + continue + } + for dir, pm := range pv.Modules { + if pm.Manifest == p.Fps[m].Manifest { + dirs[m] = dir + } + } } fps := maps.Clone(p.Fps) for _, m := range p.Modules { @@ -166,12 +185,12 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool if fps == nil { fps = map[string]delivery.CandidateModule{} } - if fps[m] != next { + if !reflect.DeepEqual(fps[m], next) { fps[m] = next changed = true } } - if changed || !maps.Equal(fps, p.Fps) { + if changed || !maps.EqualFunc(fps, p.Fps, func(a, b delivery.CandidateModule) bool { return reflect.DeepEqual(a, b) }) { p.Fps = fps } } @@ -189,6 +208,9 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive if !ok && dir == "" { pm, ok = pv.Modules[delivery.RootPath] } + if !ok && dir == delivery.RootPath { + pm, ok = pv.Modules[""] + } if !ok { return delivery.CandidateModule{None2: NonePreviewNoModule} } @@ -207,7 +229,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive out.None2 = NonePreviewNoManifest return out } - in := inputsOf(m, kept.JSON, pm.Tree, pm.Locks, ix.baseFP(ctx)) + in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, ix.baseFP(ctx)) if in.Kept == nil { out.None2 = in.None2 return out @@ -248,5 +270,8 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive return out } out.FP2 = fp + if len(inputs.Paths) > 0 { + out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4) + } return out } diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 2235218..0f2c781 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -34,7 +34,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 { // ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each // repository the source names, each module built from it whose fingerprint was computed at the head the source names is // written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped -// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. +// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The +// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6). func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error { ix.mu.Lock() if rev < ix.source.rev { @@ -46,34 +47,39 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc if ix.Modules == nil { return ix.RecomputeCandidates(ctx) } - repositories := slices.Sorted(maps.Keys(s.Heads)) - for _, repository := range repositories { + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + for _, key := range slices.Sorted(maps.Keys(held)) { + repository := "" + for r := range s.Heads { + if delivery.SameRepository(r, held[key].Repository) { + repository = r + } + } + if repository == "" { + continue + } head := s.Heads[repository] - for _, k := range ix.Builds.Modules(repository) { - key, err := delivery.ModulesKey(k.Module) - if err != nil { - ix.logf("%s: the module %q has no usable key, skipped: %v", repository, k.Module, err) - continue + err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil } - err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - cur, err := delivery.DecodeModule(value) - if err != nil { - ix.logf("modules.%s holds what is not a module, left: %v", key, err) - return nil, false, nil - } - if cur.Repository != repository || cur.Commit != head || cur.Src == rev { - return nil, false, nil - } - cur.Src = rev - data, err := delivery.EncodeModule(cur) - return data, false, err - }) + cur, err := delivery.DecodeModule(value) if err != nil { - return fmt.Errorf("modules.%s: %w", key, err) + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + return nil, false, nil } + if cur.Repository != repository || cur.Commit != head || cur.Src == rev { + return nil, false, nil + } + cur.Src = rev + data, err := delivery.EncodeModule(cur) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) } } // A trunk that moved makes the previews made onto the old one out of date. diff --git a/modules/mesh-delivery-indexer/internal/index/source_test.go b/modules/mesh-delivery-indexer/internal/index/source_test.go index 634e216..d7b16a5 100644 --- a/modules/mesh-delivery-indexer/internal/index/source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/source_test.go @@ -15,8 +15,7 @@ func sourceOf(repo, head string) delivery.Source { } func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ix, mods := modulesIndexer(t) ctx := context.Background() if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) @@ -63,7 +62,7 @@ func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.FP == "" { + if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.Path != "modules/a" { t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok) } } @@ -75,8 +74,8 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344 if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.FP == "" { - t.Fatalf("module a = %+v (%v), want a fingerprint with Src 0", m, ok) + if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.Path != "modules/a" { + t.Fatalf("module a = %+v (%v), want the merge written with Src 0", m, ok) } // A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0. if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json new file mode 100644 index 0000000..3b62d21 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-claude-code.module.json @@ -0,0 +1,249 @@ +{ + "module": "claude-code", + "version": "1", + "moved": true, + "slug": "agent", + "capabilities": [ + "package-manager" + ], + "requires": [ + "mcp-endpoint", + { + "name": "gitea-read", + "optional": true + } + ], + "binds": { + "mcp-endpoint": "${dir:state}/mcp-endpoint.json" + }, + "secrets": { + "gitea-read": "${dir:state}/gitea-read.secret" + }, + "uses": [ + "operator-channel" + ], + "state": [ + "servers", + "holdings", + "config", + "proposals", + "questions", + "memory", + "waits" + ], + "reads": [ + "claude-licence-manager.bindings" + ], + "tools": [ + "claude_code_status", + "claude_code_render", + "claude_code_guard", + "claude_code_pull", + "claude_code_grant", + "claude_code_add_api_key", + "claude_code_registrations", + "claude_code_memory", + "claude_code_mcp_list", + "claude_code_mcp_register", + "claude_code_mcp_unregister", + "claude_code_skill_list", + "claude_code_skill_register", + "claude_code_skill_unregister", + "claude_code_agent_list", + "claude_code_agent_register", + "claude_code_agent_unregister", + "claude_code_command_list", + "claude_code_command_register", + "claude_code_command_unregister", + "claude_code_hook_list", + "claude_code_hook_register", + "claude_code_hook_unregister", + "claude_code_output_style_list", + "claude_code_output_style_register", + "claude_code_output_style_unregister", + "claude_code_instructions_list", + "claude_code_instructions_register", + "claude_code_instructions_unregister", + "claude_code_instructions_propose", + "claude_code_proposals", + "claude_code_ask", + "claude_code_answer", + "claude_code_wait", + "claude_code_settings_get", + "claude_code_settings_set", + "claude_code_settings_clear", + "claude_code_permission_add", + "claude_code_permission_remove", + "claude_code_config_list", + "claude_code_config_show", + "claude_code_config_status", + "claude_code_config_import", + "claude_code_home_show", + "claude_code_home_remove", + "claude_code_home_removed", + "claude_code_home_restore", + "claude_code_judge", + "claude_code_judged" + ], + "data": { + "own": [ + { + "id": "agent-home", + "path": "${dir:agent-home}", + "class": "valuable", + "why": "the operator's agent's own files: its memory, history and projects" + } + ] + }, + "config": { + "role": { + "type": "string", + "description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.", + "trusted": true, + "default": "" + }, + "mcp_servers": { + "type": "object", + "description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.", + "trusted": true, + "default": {} + }, + "managed_settings": { + "type": "object", + "description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.", + "trusted": true, + "predefined": "agent-managed-settings", + "default": {} + }, + "instructions": { + "type": "object", + "description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "output_styles": { + "type": "object", + "description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "skills": { + "type": "object", + "description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.", + "trusted": true, + "default": {} + }, + "commands": { + "type": "object", + "description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + }, + "agents": { + "type": "object", + "description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "default": {} + } + }, + "resources": [ + { + "id": "package", + "type": "package", + "package": "claude-code" + }, + { + "id": "managed", + "type": "directory", + "path": "/etc/claude-code", + "mode": "0755", + "unknown": "program" + }, + { + "id": "start", + "type": "file", + "path": "/usr/local/bin/claude-agent", + "mode": "0755", + "template": "files/claude-agent" + }, + { + "id": "agent-account-name", + "type": "file", + "path": "/etc/claude-code/agent-account", + "mode": "0644", + "template": "files/agent-account" + }, + { + "id": "agent-account", + "type": "user", + "name": "${machine:agent-account}", + "home": "${machine:agent-home}", + "root": "${machine:agent-root}" + }, + { + "id": "agent-home", + "type": "directory", + "path": "${machine:agent-home}/.claude", + "mode": "0700", + "owner": "${machine:agent-account}", + "unknown": "module" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "${machine:account}", + "place": ".", + "unknown": "program" + }, + { + "id": "facts", + "type": "file", + "path": "${dir:state}/facts.json", + "mode": "0600", + "owner": "${machine:account}", + "template": "files/facts.json" + }, + { + "id": "settings", + "type": "file", + "path": "${dir:state}/settings.json", + "mode": "0600", + "owner": "${machine:account}", + "takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"], + "merge": "json", + "template": "files/settings.json" + } + ], + "items": [ + { + "collection": "shell:zsh:normal", + "template": "files/zsh-normal" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/claude-code", + "binary": "claude-code", + "loads": [ + "claude-code" + ], + "env": { + "MESH_CLAUDE_CODE_STATE": "${dir:state}", + "MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json", + "MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json new file mode 100644 index 0000000..0c6983b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-dunst.module.json @@ -0,0 +1,123 @@ +{ + "module": "dunst", + "version": "1", + "moved": true, + "capabilities": [ + "package-manager" + ], + "claims": [ + { + "name": "node-notifier", + "scope": "node", + "serves": [ + "send", + "history" + ] + } + ], + "tools": [ + "dunst_pause", + "dunst_resume", + "dunst_close_all", + "dunst_rules", + "dunst_count", + "dunst_reload" + ], + "config": { + "font-size": { + "type": "number", + "description": "The size of a notification's text, in points of Inter: 10 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.", + "minimum": 6, + "maximum": 72, + "predefined": "font-size", + "default": 10 + }, + "width": { + "type": "number", + "description": "The width of a notification, in pixels: 250 fits a title of about forty characters at 10 points, and a larger font needs a wider notification.", + "minimum": 50, + "maximum": 4000, + "default": 250 + } + }, + "resources": [ + { + "id": "package", + "type": "package", + "package": "dunst" + }, + { + "id": "client", + "type": "package", + "package": "libnotify" + }, + { + "id": "configuration-dir", + "type": "directory", + "path": "${machine:account-home}/.config/dunst", + "owner": "${machine:account}", + "mode": "0755", + "unknown": "move-out" + }, + { + "id": "dropins", + "type": "directory", + "path": "${machine:account-home}/.config/dunst/dunstrc.d", + "owner": "${machine:account}", + "mode": "0755", + "unknown": "move-out", + "knows": ["50-slack.conf"] + }, + { + "id": "configuration", + "trusted": false, + "type": "file", + "path": "${machine:account-home}/.config/dunst/dunstrc", + "owner": "${machine:account}", + "mode": "0644", + "template": "files/dunstrc" + }, + { + "id": "notifier", + "type": "service", + "unit": "dunst.service", + "scope": "user", + "user": "${machine:account}", + "reload-on": [ + "configuration" + ] + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/dunst-tools", + "binary": "dunst-tools", + "loads": [ + "dunst-tools" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + }, + "emits": [ + "action-chosen", + "started", + "paused" + ], + "items": [ + { + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/50-dunst.conf" + } + ] +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json new file mode 100644 index 0000000..b65be8e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-835bf22b-xterm.module.json @@ -0,0 +1,236 @@ +{ + "module": "xterm", + "version": "1", + "moved": true, + "capabilities": [ + "package-manager" + ], + "requires": [ + "x11-display" + ], + "claims": [ + { + "name": "node-terminal-emulator", + "scope": "node", + "serves": [ + "open" + ] + } + ], + "tools": [ + "xterm_font", + "xterm_colours" + ], + "config": { + "face": { + "type": "string", + "description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).", + "pattern": "[A-Za-z0-9 ._-]{1,64}", + "default": "JetBrainsMono Nerd Font" + }, + "size": { + "type": "number", + "description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.", + "minimum": 6, + "maximum": 72, + "predefined": "font-size", + "default": 12 + }, + "save-lines": { + "type": "number", + "description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.", + "minimum": 0, + "maximum": 1000000, + "default": 10000 + }, + "term-name": { + "type": "string", + "description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.", + "pattern": "[A-Za-z0-9._-]{1,64}", + "default": "xterm-256color" + }, + "render-font": { + "type": "boolean", + "description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.", + "default": true + }, + "bold-mode": { + "type": "boolean", + "description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.", + "default": false + }, + "always-bold-mode": { + "type": "boolean", + "description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.", + "default": false + }, + "allow-bold-fonts": { + "type": "boolean", + "description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.", + "default": true + }, + "background": { + "type": "string", + "description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#000000" + }, + "foreground": { + "type": "string", + "description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#a8a8a8" + }, + "color0": { + "type": "string", + "description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#000000" + }, + "color1": { + "type": "string", + "description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#9e1828" + }, + "color2": { + "type": "string", + "description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#008800" + }, + "color3": { + "type": "string", + "description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#d2bb4b" + }, + "color4": { + "type": "string", + "description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#414171" + }, + "color5": { + "type": "string", + "description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#963c59" + }, + "color6": { + "type": "string", + "description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#418179" + }, + "color7": { + "type": "string", + "description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#bebebe" + }, + "color8": { + "type": "string", + "description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#666666" + }, + "color9": { + "type": "string", + "description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#bc5766" + }, + "color10": { + "type": "string", + "description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#61a171" + }, + "color11": { + "type": "string", + "description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#e7db52" + }, + "color12": { + "type": "string", + "description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#5085af" + }, + "color13": { + "type": "string", + "description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#a97a99" + }, + "color14": { + "type": "string", + "description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#6ba4a4" + }, + "color15": { + "type": "string", + "description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#ffffff" + }, + "copy-binding": { + "type": "string", + "description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.", + "pattern": "[^:\\\\\\n]+", + "default": "Ctrl Shift ~Alt C" + }, + "paste-binding": { + "type": "string", + "description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.", + "pattern": "[^:\\\\\\n]+", + "default": "Ctrl Shift ~Alt V" + } + }, + "items": [ + { + "collection": "environment", + "name": "terminal", + "attributes": { + "kind": "variable", + "name": "TERMINAL" + }, + "template": "files/env-terminal" + }, + { + "collection": "shell:xresources:normal", + "name": "xresources-normal", + "template": "files/xresources-normal" + } + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "xterm" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/xterm-tools", + "binary": "xterm-tools", + "loads": [ + "xterm-tools" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json new file mode 100644 index 0000000..404139a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json @@ -0,0 +1,16 @@ +{ + "module": "mesh-sdk", + "version": "1", + "slug": "sdk", + "build": { + "artifacts": [ + { + "name": "lib", + "kind": "package", + "language": "typescript", + "paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"] + } + ] + }, + "resources": [] +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index ab29381..0257f1b 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -134,6 +134,9 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWit // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error { + if err := ix.fillPaths(ctx); err != nil { + return err + } if err := ix.recomputeModules(ctx, overlay); err != nil { return err } @@ -366,7 +369,7 @@ func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Man if err != nil { return catalogue.Manifest{}, err } - return catalogue.ParseManifest([]byte(kept.JSON)) + return catalogue.ParseManifestForInputs([]byte(kept.JSON)) } // hasContext is whether an artifact of the manifest reads another repository's tree. @@ -420,3 +423,43 @@ func (ix *Indexer) RecomputeForBuild(ctx context.Context) error { } return ix.Recompute(ctx) } + +// fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its +// newest build record names, once, by compare-and-set, changing nothing else (issue 1614): the builder needs it, and a +// rarely merged repository would leave it without one for weeks. A key that names a directory is never written here. +// This read of the controller's build records goes in R4g with build-agent. +func (ix *Indexer) fillPaths(ctx context.Context) error { + if ix.Modules == nil || ix.Builds == nil { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + for _, key := range slices.Sorted(maps.Keys(held)) { + if held[key].Path != "" { + continue + } + dir, ok := ix.Builds.PathOf(key, held[key].Repository) + if !ok { + continue + } + err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil + } + cur, err := delivery.DecodeModule(value) + if err != nil || cur.Path != "" { + return nil, false, nil + } + cur.Path = dir + data, err := delivery.EncodeModule(cur) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } + ix.logf("modules.%s names no directory: %q filled from its newest build record (issue 1614)", key, dir) + } + return nil +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 055782f..ce9784a 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -38,6 +38,7 @@ type manifest struct { Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` + Invokes []string `json:"invokes"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -93,8 +94,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if m.SecretsOwner != "mesh-delivery-indexer" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // No invokes, no emits, no streams, no bus, no seat, no tool (ADR 0437 §4). - for _, k := range []string{"invokes", "streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + // One invoke: the forge's trees with the named paths' objects, since the indexer is the one reader of a manifest's paths + // (ADR 0499 clauses 2 and 3, ADR 0495 clause 1, the lead's decision L8 of R4e). + if want := []string{"gitea.gitea_commit_trees"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + // No emits, no streams, no bus, no seat, no tool (ADR 0437 §4). + for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the indexer has none", k) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index cb58b8b..706ee06 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -20,6 +20,9 @@ "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], + "invokes": [ + "gitea.gitea_commit_trees" + ], "own-secrets": { "broker": "${dir:state}/broker" }, diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index b6079de..645c726 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -35,18 +35,39 @@ func Tools(r view.Reader) []stdio.Tool { mods, err := r.Modules(only) return map[string]any{"modules": mods}, err }}, + {Name: view.CandidateReadTool, + Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates., as stored: " + + "the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " + + "they have none (none2). Read-only.", + Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "the candidate's group, the branch name its pull requests share"}}}, + Run: func(args map[string]any) (any, error) { + group, _ := args["group"].(string) + return r.Candidate(group) + }}, + {Name: view.CandidatesReadTool, + Description: "Every candidate the indexer holds, by group, each as stored under candidates.; with module, only those with a " + + "pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only candidates touching this module"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + c, bad, err := r.Candidates(module) + return map[string]any{"candidates": c, "unreadable": bad}, err + }}, {Name: view.BuilderBuildsReadTool, Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " + "(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " + - "or state (queued, building, done, failed). Read-only.", + "or state (queued, building, done, failed); at most limit builds (default 50), newest revision first for a module, and the answer says when it was cut and lists keys that are no build key (unreadable). Without a module, give one to get its newest. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "only this module's builds"}, - "state": map[string]any{"type": "string", "description": "only builds in this state"}}}, + "state": map[string]any{"type": "string", "description": "only builds in this state"}, + "limit": map[string]any{"type": "integer", "description": "at most this many builds (default 50), newest revision first"}}}, Run: func(args map[string]any) (any, error) { module, _ := args["module"].(string) state, _ := args["state"].(string) - rows, err := r.BuilderBuilds(module, state) - return map[string]any{"builds": rows}, err + limit, _ := args["limit"].(float64) + return r.BuilderBuilds(module, state, int(limit)) }}, {Name: view.BuilderBuildReadTool, Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " + @@ -65,7 +86,13 @@ func Tools(r view.Reader) []stdio.Tool { } func read(state stdio.KeptState) view.StateRead { - return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) { + return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) { + e, err := state.Get(key) + if err != nil || e == nil { + return nil, 0, err + } + return e.Value, e.Revision, nil + }, Get: func(key string) (json.RawMessage, error) { e, err := state.Get(key) if err != nil || e == nil { return nil, err @@ -75,7 +102,7 @@ func read(state stdio.KeptState) view.StateRead { } func main() { - r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))} + r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))} if err := stdio.Serve("", Tools(r)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index a5e2193..0c13bdf 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "strings" "testing" @@ -117,13 +118,16 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) { if err != nil { t.Fatal(err) } - rows := out.(map[string]any)["builds"].([]view.BuilderRow) + rows := out.(view.BuildsAnswer).Builds + if u := out.(view.BuildsAnswer).Unreadable; len(u) != 1 || u[0] != "junk" { + t.Errorf("unreadable = %v, want [junk] (issue 1818)", u) + } if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild { t.Errorf("rows = %+v", rows) } for args, want := range map[string]string{"state": keyB, "module": keyA} { out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]}) - got := out.(map[string]any)["builds"].([]view.BuilderRow) + got := out.(view.BuildsAnswer).Builds if len(got) != 1 || got[0].Key != want { t.Errorf("%s filter = %+v, want %s", args, got, want) } @@ -156,3 +160,107 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) { t.Errorf("an unbuilt fingerprint: %v", err) } } + +// bigBuilds is a bucket of n keys over 100 modules that counts the values read. +type bigBuilds struct { + fake fake + reads *int +} + +func newBig(n int) bigBuilds { + f := fake{} + for i := 0; i < n; i++ { + f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild + } + return bigBuilds{fake: f, reads: new(int)} +} + +func (b bigBuilds) reader() view.Reader { + r := b.fake.reader() + get := r.State.Get + r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) }, + GetEntry: func(k string) (json.RawMessage, uint64, error) { + *b.reads++ + raw, err := get(k) + var rev uint64 + fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number + return raw, rev + 1, err + }} + return r +} + +// 1827/AC1: with 2,000 keys, a module's builds are found by key prefix, so only that module's values are read (20, not 2,000), +// newest revision first, within the limit, and the answer says it was cut. +func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) { + b := newBig(2000) + out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"module": "mod07", "limit": float64(5)}) + if err != nil { + t.Fatal(err) + } + a := out.(view.BuildsAnswer) + if *b.reads != 20 { + t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads) + } + if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") { + t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note) + } + for i := 1; i < len(a.Builds); i++ { + if a.Builds[i-1].Revision < a.Builds[i].Revision { + t.Errorf("not newest first: %d before %d", a.Builds[i-1].Revision, a.Builds[i].Revision) + } + } + if a.Builds[0].Revision != 1908 { + t.Errorf("newest revision = %d, want 1908", a.Builds[0].Revision) + } +} + +// 1827/AC2: with no module the answer reads at most the limit's worth in key order (default 50), never the 2,000. +func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) { + b := newBig(2000) + out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{}) + if err != nil { + t.Fatal(err) + } + a := out.(view.BuildsAnswer) + if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit { + t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads) + } + b = newBig(2000) + out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"}) + if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 { + t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut) + } +} + +// 1818: fp2 together with commit is refused, naming both. +func TestBuilderBuildRefusesFP2WithCommit(t *testing.T) { + r := both(fake{keyA: doneBuild}, fake{"netcheck": `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}`}) + _, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2A, "commit": "c1"}) + if err == nil || !strings.Contains(err.Error(), "fp2") || !strings.Contains(err.Error(), "commit") || !strings.Contains(err.Error(), "not both") { + t.Errorf("err = %v", err) + } +} + +// 1681: indexer_candidates and indexer_candidate answer the stored value; a pull request's fp2 per touched module shows. +func TestCandidateToolsAnswerTheStoredValue(t *testing.T) { + cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}` + other := `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}` + r := fake{"feat/x": cand, "fix": other, "bad": "{"}.reader() + r.CandidatesState = r.State + out, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "feat/x"}) + if err != nil || string(out.(json.RawMessage)) != cand || !strings.Contains(string(out.(json.RawMessage)), fp2B) { + t.Fatalf("candidate = %v, %v", out, err) + } + if _, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "gone"}); err == nil || !strings.Contains(err.Error(), "no candidate gone") { + t.Errorf("err = %v", err) + } + out, err = run(t, r, view.CandidatesReadTool, map[string]any{}) + m := out.(map[string]any) + if err != nil || len(m["candidates"].(map[string]json.RawMessage)) != 2 || len(m["unreadable"].([]string)) != 1 { + t.Errorf("all = %v, %v", m, err) + } + out, _ = run(t, r, view.CandidatesReadTool, map[string]any{"module": "zsh"}) + if c := out.(map[string]any)["candidates"].(map[string]json.RawMessage); len(c) != 1 || c["feat/x"] == nil { + t.Errorf("by module = %v", c) + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 4f3cf6c..4452978 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -136,7 +136,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket} { + for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket} { go follow(ctx, js, bucket, model, logger) } select { diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index b8f5dd8..1d89b48 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 3686111..db1ea18 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,13 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337 h1:cZyFnvDiBpnNx8ifSUG9WA61OHsn7PC2ml2s4eD7AkA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a h1:Vjk2Kq+g/Q86N8CCHq3OatJIJ0JSy6Kh980ekGxNLmE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go b/modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go new file mode 100644 index 0000000..97fb6d1 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/fingerprints_newform_test.go @@ -0,0 +1,53 @@ +package view + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// I5 (issue 1851): from R4e the indexer writes the new form alone, and a module's fingerprint is built by the builder's key +// of it; the page shows the indexer's fp2 against that key's state, or the reason it has none. + +const fp2A = "src2:3333333333333333333333333333333333333333333333333333333333333333" + +func builderKey(t *testing.T, module, fp2 string, b delivery.BuilderBuild) Entry { + t.Helper() + key, err := delivery.BuilderBuildKey(module, fp2) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.BuilderBuildsBucket, Key: key, Value: data, At: t0} +} + +func TestAFingerprintOfTheNewFormIsShownAgainstItsBuilderKey(t *testing.T) { // I5 + m := New() + manifest := strings.Repeat("a", 64) + d := "sha256:" + strings.Repeat("2", 64) + fill(m, + modEntry(t, "a", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}), + builderKey(t, "a", fp2A, delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.mesh-delivery-builder", Manifest: manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "program", Kind: "bundle", Digest: d, Reference: "artifact-store://a/program/blobs/" + d}}}), + modEntry(t, "f", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}), + builderKey(t, "f", fp2A, delivery.BuilderBuild{State: delivery.BuildFailed, By: "shanks.mesh-delivery-builder", Manifest: manifest, Error: "not stored: program: 507"}), + modEntry(t, "n", delivery.Module{FP2: fp2A, Manifest: manifest, Repository: "novox/mesh-delivery", Commit: hA}), + modEntry(t, "r", delivery.Module{None2: "toolchain not yet reported", Repository: "novox/mesh-delivery", Commit: hA})) + s := m.Snapshot() + if r := fpRow(t, s, "a"); r.State != FPBuilt || r.Indexer != fp2A { + t.Errorf("a = %+v, want built with the indexer's fp2", r) + } + if r := fpRow(t, s, "f"); r.State != FPBuildFailed || !strings.Contains(r.Reason, "not stored") { + t.Errorf("f = %+v, want build failed with its error", r) + } + if r := fpRow(t, s, "n"); r.State != FPNoKey || r.Indexer != fp2A { + t.Errorf("n = %+v, want no key", r) + } + if r := fpRow(t, s, "r"); r.State != FPNone || r.Reason != "toolchain not yet reported" { + t.Errorf("r = %+v, want none with the new form's reason", r) + } +} diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index e99a327..8f1546a 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -7,6 +7,7 @@ package view import ( "encoding/json" "fmt" + "slices" "sort" "strings" @@ -23,12 +24,16 @@ const ( type StateRead struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, error) + // GetEntry, when set, also gives the revision of the value; builder_builds orders by it. + GetEntry func(key string) (json.RawMessage, uint64, error) } // Reader answers the read tools from a modules state. type Reader struct { State StateRead // the indexer's modules Builds StateRead // the builder's builds + + CandidatesState StateRead // the indexer's candidates } // Module is the stored value of modules., exactly as the indexer wrote it. @@ -84,6 +89,67 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) { return out, nil } +// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates., as stored, so the +// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live. +const ( + CandidateReadTool = "indexer_candidate" + CandidatesReadTool = "indexer_candidates" +) + +// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it. +func (r Reader) Candidate(group string) (json.RawMessage, error) { + key, err := delivery.CandidatesKey(strings.TrimSpace(group)) + if err != nil { + return nil, err + } + if r.CandidatesState.Get == nil { + return nil, fmt.Errorf("the candidates state cannot be read here") + } + raw, err := r.CandidatesState.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no candidate %s in the candidates state", group) + } + return raw, nil +} + +// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it. +// A value that is not readable is listed under unreadable with its key, not skipped. +func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) { + if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil { + return nil, nil, fmt.Errorf("the candidates state cannot be read here") + } + keys, err := r.CandidatesState.Keys() + if err != nil { + return nil, nil, err + } + sort.Strings(keys) + out := map[string]json.RawMessage{} + var unreadable []string + for _, k := range keys { + raw, err := r.CandidatesState.Get(k) + if err != nil { + return nil, nil, err + } + if len(raw) == 0 { + continue + } + group, perr := delivery.ParseCandidatesKey(k) + var c delivery.Candidate + if perr != nil || json.Unmarshal(raw, &c) != nil { + unreadable = append(unreadable, k) + continue + } + if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) { + continue + } + out[group] = raw + } + return out, unreadable, nil +} + // The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `.` // in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live. const ( @@ -91,48 +157,100 @@ const ( BuilderBuildsReadTool = "builder_builds" ) -// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value -// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error). +// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its +// stored value (when the read gives one) and the stored value as the builder wrote it. type BuilderRow struct { Key string `json:"key"` Module string `json:"module"` Fingerprint string `json:"fingerprint"` + Revision uint64 `json:"revision,omitempty"` Build json.RawMessage `json:"build"` } +// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call. +// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one). +const ( + DefaultBuildsLimit = 50 + MaxBuildReads = 200 +) + +// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and +// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818). +type BuildsAnswer struct { + Builds []BuilderRow `json:"builds"` + Cut bool `json:"cut,omitempty"` + Note string `json:"note,omitempty"` + Unreadable []string `json:"unreadable,omitempty"` +} + +func (r Reader) get(key string) (json.RawMessage, uint64, error) { + if r.Builds.GetEntry != nil { + return r.Builds.GetEntry(key) + } + raw, err := r.Builds.Get(key) + return raw, 0, err +} + func (r Reader) builderRow(key string) (BuilderRow, bool, error) { module, fp, err := delivery.ParseBuilderBuildKey(key) if err != nil { return BuilderRow{}, false, nil // a key that is no build key is not a build } - raw, err := r.Builds.Get(key) + raw, rev, err := r.get(key) if err != nil { return BuilderRow{}, false, err } if len(raw) == 0 { return BuilderRow{}, false, nil } - return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil + return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil } -// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds -// of that module or in that state. -func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) { +// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys +// that begin `.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit +// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the +// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the +// answer is the first rows found, not the newest of the bucket: give a module to get its newest. +func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) { if r.Builds.Keys == nil || r.Builds.Get == nil { - return nil, fmt.Errorf("the builder's builds cannot be read here") + return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here") + } + if limit <= 0 { + limit = DefaultBuildsLimit } keys, err := r.Builds.Keys() if err != nil { - return nil, err + return BuildsAnswer{}, err } sort.Strings(keys) - rows := []BuilderRow{} + prefix := "" + if module = strings.TrimSpace(module); module != "" { + prefix = module + "." + } + out := BuildsAnswer{Builds: []BuilderRow{}} + var candidates []string for _, k := range keys { + if prefix != "" && !strings.HasPrefix(k, prefix) { + continue + } + if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil { + out.Unreadable = append(out.Unreadable, k) + continue + } + candidates = append(candidates, k) + } + reads := 0 + for _, k := range candidates { + if reads >= MaxBuildReads { + out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads) + break + } + reads++ row, ok, err := r.builderRow(k) if err != nil { - return nil, err + return BuildsAnswer{}, err } - if !ok || (module != "" && row.Module != module) { + if !ok { continue } if state != "" { @@ -143,9 +261,26 @@ func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) { continue } } - rows = append(rows, row) + out.Builds = append(out.Builds, row) + // Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them. + if prefix == "" && len(out.Builds) >= limit { + if reads < len(candidates) { + out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit) + } + break + } } - return rows, nil + sort.SliceStable(out.Builds, func(i, j int) bool { + if out.Builds[i].Revision != out.Builds[j].Revision { + return out.Builds[i].Revision > out.Builds[j].Revision + } + return out.Builds[i].Key < out.Builds[j].Key + }) + if len(out.Builds) > limit { + out.Builds = out.Builds[:limit] + out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module) + } + return out, nil } // BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint @@ -156,6 +291,9 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) { if r.Builds.Get == nil { return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here") } + if fp2 != "" && commit != "" { + return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit) + } if fp2 == "" { raw, err := r.Module(module) if err != nil { diff --git a/modules/mesh-delivery-view/internal/view/serve.go b/modules/mesh-delivery-view/internal/view/serve.go index 3678c6c..d36b09d 100644 --- a/modules/mesh-delivery-view/internal/view/serve.go +++ b/modules/mesh-delivery-view/internal/view/serve.go @@ -146,6 +146,10 @@ func fingerprintText(s Snapshot) [][]string { what = f.Reason case FPNoBuild: what = "indexer " + f.Indexer + "; no successful trunk build recorded a fingerprint" + case FPBuildFailed, FPBuilding: + what = "indexer " + f.Indexer + "; builder: " + f.Reason + case FPNoKey: + what = "indexer " + f.Indexer + "; the builder holds no key of it" } out = append(out, []string{f.Module, f.State, what}) } diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index ad38cef..2cebc2e 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -101,6 +101,12 @@ const ( FPDiffers = "differs" FPNone = "none" FPNoBuild = "no build" + // The states of a fingerprint of the new form against the builder's key of it (I5, issue 1851): built (the key is done), + // build failed (Reason is its error), building (queued, claimed or building; Reason says what it waits for) and no key. + FPBuilt = "built" + FPBuildFailed = "build failed" + FPBuilding = "building" + FPNoKey = "no key" ) // FingerprintRow compares the indexer's fingerprint of a module at a trunk merge with the source fingerprint its newest @@ -147,6 +153,7 @@ type Model struct { builds map[string]buildEntry // by .build. groups map[string]groupEntry // the indexer's candidates, by group modules map[string]moduleEntry // the indexer's modules, by module + keys map[string]keyEntry // the builder's build keys, by . asks map[string]askRecord // the gate's asks, by key answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable) newest map[string]string // module -> id @@ -209,6 +216,11 @@ type groupEntry struct { err string } +type keyEntry struct { + value delivery.BuilderBuild + err string +} + type moduleEntry struct { value delivery.Module err string @@ -233,7 +245,7 @@ type nodeEntry struct { // New is an empty model. func New() *Model { - return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, asks: map[string]askRecord{}, answers: map[string]string{}, newest: map[string]string{}, + return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, answers: map[string]string{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}} } @@ -321,6 +333,17 @@ func (m *Model) Apply(e Entry) { id, _, _ = strings.Cut(a.Warrant, "@") } m.answers[e.Key] = id + case delivery.BuilderBuildsBucket: + if e.Deleted { + delete(m.keys, e.Key) + return + } + b, err := delivery.DecodeBuilderBuild(e.Value) + if err != nil { + m.keys[e.Key] = keyEntry{err: err.Error()} + return + } + m.keys[e.Key] = keyEntry{value: b} case BuildsBucket: module, rest, ok := strings.Cut(e.Key, ".") if !ok { @@ -525,6 +548,11 @@ func (m *Model) fingerprintRows() []FingerprintRow { continue } row := FingerprintRow{Module: name, Repository: mod.value.Repository, Commit: mod.value.Commit} + if mod.value.FP2 != "" || mod.value.None2 != "" { + // The new form (R4e): the indexer's fp2 against the builder's key of it. + out = append(out, m.newFormRow(name, row, mod.value)) + continue + } b, hasBuild := newest[name] switch { case mod.value.None != "": @@ -542,6 +570,35 @@ func (m *Model) fingerprintRows() []FingerprintRow { return out } +// newFormRow is a module's row of the new form: none with its reason, or its fp2 against the state of the builder's key +// of it (I5, issue 1851; the build records name no fingerprint of the new form). +func (m *Model) newFormRow(name string, row FingerprintRow, v delivery.Module) FingerprintRow { + if v.FP2 == "" { + row.State, row.Reason = FPNone, v.None2 + return row + } + row.Indexer = v.FP2 + key, err := delivery.BuilderBuildKey(name, v.FP2) + if err != nil { + row.State, row.Reason = FPNoKey, err.Error() + return row + } + k, ok := m.keys[key] + switch { + case !ok: + row.State = FPNoKey + case k.err != "": + row.State, row.Reason = FPBuildFailed, "the key cannot be read: "+k.err + case k.value.State == delivery.BuildDone: + row.State, row.Build = FPBuilt, v.FP2 + case k.value.State == delivery.BuildFailed: + row.State, row.Reason = FPBuildFailed, k.value.Error + default: + row.State, row.Reason = FPBuilding, strings.TrimSpace(k.value.State+" "+k.value.Waits) + } + return row +} + // count is the number of items of a carried JSON array or object, and 0 for anything else. func count(raw json.RawMessage) int { var list []json.RawMessage diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 0e185a2..b3c0e1c 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 89bf62c..09cd55b 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -6,6 +6,8 @@ "tools": [ "indexer_module", "indexer_modules", + "indexer_candidate", + "indexer_candidates", "builder_build", "builder_builds" ],