diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 7f8bc8e2..7570fb79 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it // now waits for a gate like any other build, and the release plan walks it through the machines, one // at a time, each judged. - fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+ + fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+ "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on)) return nil } diff --git a/cmd/mesh-controller/walk_said.go b/cmd/mesh-controller/walk_said.go index ddb724e4..3e5d9bad 100644 --- a/cmd/mesh-controller/walk_said.go +++ b/cmd/mesh-controller/walk_said.go @@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s reached = appendNew(reached, r.Gate.Machines...) carried(r.Gate) } + for _, g := range r.Gates { + if g != nil && !g.Pending { + reached = appendNew(reached, g.Machines...) + carried(g) + } + } for _, tier := range p.Tiers { for _, m := range tier { for _, n := range running(m) { diff --git a/internal/builder/builder.go b/internal/builder/builder.go index c2c1cbf9..728ce83e 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -1088,6 +1088,16 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err) } say("image", "built, publishing") + // Its bases linked into its repository first, so the push needs none of their blobs from this + // machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push + // from a store that holds them works without, and one that fails says which blob it lacked. + if lender, can := publish.(BaseLender); can { + if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil { + say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err) + } else if n > 0 { + say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name) + } + } reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err @@ -1439,6 +1449,19 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin return args, resolved, nil } +// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them. +func buildArgValues(args []string) []string { + var values []string + for i := 0; i+1 < len(args); i += 2 { + if args[i] == "--build-arg" { + if _, value, has := strings.Cut(args[i+1], "="); has { + values = append(values, value) + } + } + } + return values +} + // compile runs a module's own code through its toolchain, and says where the result is. // // **In the module's own directory, under the path the toolchain expects.** A module is compiled diff --git a/internal/builder/check.go b/internal/builder/check.go index bf0ce5bc..a2107499 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -103,6 +103,23 @@ type CheckSpec struct { // for the walk to take. None for a group's composed check, whose members' own checks build. Builds []CheckBuildSpec BuildWith *CheckBuilding + // Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull + // request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read. + Tests string + StoreOnly bool +} + +// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing +// for a pull request's check, which always runs whole. +func selection(spec CheckSpec) []string { + var env []string + if spec.Tests != "" { + env = append(env, EnvTests+"="+spec.Tests) + } + if spec.StoreOnly { + env = append(env, EnvStoreOnly+"=1") + } + return env } // GroupHead is one other head of a delivery group's composed check. @@ -193,6 +210,11 @@ const ( // EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a // pull request checked alone. EnvGroup = "MESH_CHECK_GROUP" + // EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request + // asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's + // check. A script that reads neither runs whole. + EnvTests = "MESH_CHECK_TESTS" + EnvStoreOnly = "MESH_CHECK_STORE_ONLY" ) // CheckTimeout bounds one check; a check that runs past it is an error, not a pass. @@ -515,6 +537,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","), EnvGroup + "=" + groupFile, "GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")} + env = append(env, selection(spec)...) v := CheckVerdict{} modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...) diff --git a/internal/builder/check_test.go b/internal/builder/check_test.go index 92583f5a..1eb5890a 100644 --- a/internal/builder/check_test.go +++ b/internal/builder/check_test.go @@ -590,3 +590,15 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) } } + +// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a +// pull request's check is handed nothing, and runs whole. +func TestOnlyATrysSelectionReachesTheScript(t *testing.T) { + if env := selection(CheckSpec{Number: 7}); len(env) != 0 { + t.Fatalf("a pull request's check was handed a selection: %v", env) + } + env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true}) + if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" { + t.Fatalf("a try's selection: %v", env) + } +} diff --git a/internal/builder/lend.go b/internal/builder/lend.go new file mode 100644 index 00000000..d3984647 --- /dev/null +++ b/internal/builder/lend.go @@ -0,0 +1,178 @@ +package builder + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "runtime" + "strings" +) + +// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694). +// +// A push reads every blob of the image out of this machine's store, unless the repository it goes to +// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only +// what the registry lacks. A runtime keeping its images in containerd's store — the default for a +// fresh install from docker 29 on — does not always hold the compressed layers of the base an image +// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an +// image then fails on a layer this machine never had ("not all of them are available locally"), while +// the very bytes sit in this registry, under the repository the base was mirrored into. +// +// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once +// the repository holds a blob the push asks this machine for nothing of it — whichever store the +// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push +// finds the same blobs already there. + +// BaseLender links what a build's bases hold into the repository its image is about to be pushed to. +type BaseLender interface { + // LendBases mounts every blob of each base this registry holds — for the platform this machine + // builds — into `repository`, and answers how many it mounted. + LendBases(ctx context.Context, repository string, bases []string) (int, error) +} + +// platformed is an index's entry: a manifest and the platform it is for. +type platformed struct { + descriptor + Platform *struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` +} + +// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its +// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it +// from this machine's store as before. +func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) { + mounted := 0 + for _, base := range bases { + name, digest, pinned := strings.Cut(base, "@") + if !pinned || !strings.HasPrefix(digest, "sha256:") { + continue + } + where, err := parseReference(name + "@" + digest) + if err != nil { + continue + } + from := where.repository + if from == repository { + continue + } + blobs, held, err := r.blobsOf(ctx, from, digest) + if err != nil { + return mounted, fmt.Errorf("reading %s: %w", base, err) + } + if !held { + continue + } + for _, blob := range blobs { + did, err := r.mount(ctx, repository, from, blob) + if err != nil { + return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err) + } + if did { + mounted++ + } + } + } + return mounted, nil +} + +// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each +// manifest it names for this machine's platform. False when the registry holds no such manifest there. +func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) { + body, held, err := r.manifest(ctx, repository, digest) + if err != nil || !held { + return nil, held, err + } + var document struct { + Manifests []platformed `json:"manifests"` + Config *descriptor `json:"config"` + Layers []descriptor `json:"layers"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err) + } + var blobs []string + if document.Config != nil { + blobs = append(blobs, document.Config.Digest) + } + for _, l := range document.Layers { + blobs = append(blobs, l.Digest) + } + for _, m := range document.Manifests { + if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH { + continue + } + more, held, err := r.blobsOf(ctx, repository, m.Digest) + if err != nil { + return nil, false, err + } + if held { + blobs = append(blobs, more...) + } + } + return blobs, true, nil +} + +// manifest reads one manifest document from this registry; false when it is not there. +func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, + "http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil) + if err != nil { + return nil, false, err + } + request.Header.Set("Accept", manifestAccept) + response, err := r.client().Do(request) + if err != nil { + return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err) + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return nil, false, nil + } + if response.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status) + } + body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + return body, err == nil, err +} + +// mount links one blob from one repository into another; false when the repository already held it. +func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) { + base := "http://" + r.Address + "/v2/" + repository + if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil { + return false, err + } else if there { + return false, nil + } + start, err := http.NewRequestWithContext(ctx, http.MethodPost, + base+"/blobs/uploads/?mount="+digest+"&from="+from, nil) + if err != nil { + return false, err + } + begun, err := r.client().Do(start) + if err != nil { + return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err) + } + begun.Body.Close() + if begun.StatusCode == http.StatusCreated { + return true, nil + } + if begun.StatusCode == http.StatusAccepted { + // Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open. + if location := begun.Header.Get("Location"); location != "" { + if strings.HasPrefix(location, "/") { + location = "http://" + r.Address + location + } + if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil { + if done, err := r.client().Do(cancel); err == nil { + done.Body.Close() + } + } + } + return false, fmt.Errorf("%s did not mount it from %s", r.Address, from) + } + return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from) +} diff --git a/internal/builder/lend_test.go b/internal/builder/lend_test.go new file mode 100644 index 00000000..ff92a998 --- /dev/null +++ b/internal/builder/lend_test.go @@ -0,0 +1,140 @@ +package builder + +import ( + "context" + "encoding/json" + "net/http/httptest" + "runtime" + "strings" + "testing" +) + +// A built image's bases are linked into its repository before it is pushed (novox/hq issue 694): a +// runtime keeping containerd's image store may not hold a base layer it never had to fetch, and its push +// asks this machine only for what the repository lacks. + +// aBaseForThisMachine puts an index into the registry under `repository`: one manifest for this +// machine's platform, one for another, and answers the index's digest and each manifest's blobs. +func aBaseForThisMachine(t *testing.T, dst *aRegistryOfRepositories, repository string) (string, []string, []string) { + t.Helper() + put := func(arch string) (string, []string) { + config := []byte(`{"architecture":"` + arch + `"}`) + layer := []byte(arch + " layer bytes") + for _, b := range [][]byte{config, layer} { + dst.blobs[digestOf(b)] = b + dst.link(repository, digestOf(b)) + } + m, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaManifestOCI, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digestOf(layer), "size": len(layer)}}, + }) + dst.manifests[repository+"@"+digestOf(m)] = m + return digestOf(m), []string{digestOf(config), digestOf(layer)} + } + other := "s390x" + if runtime.GOARCH == other { + other = "amd64" + } + mine, mineBlobs := put(runtime.GOARCH) + theirs, theirBlobs := put(other) + index, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaIndexOCI, + "manifests": []map[string]any{ + {"mediaType": mediaManifestOCI, "digest": mine, "size": 1, "platform": map[string]string{"os": runtime.GOOS, "architecture": runtime.GOARCH}}, + {"mediaType": mediaManifestOCI, "digest": theirs, "size": 1, "platform": map[string]string{"os": "linux", "architecture": other}}, + }, + }) + dst.manifests[repository+"@"+digestOf(index)] = index + return digestOf(index), mineBlobs, theirBlobs +} + +func TestABuildsBaseIsLinkedIntoItsRepositoryBeforeThePush(t *testing.T) { + dst := newRegistryOfRepositories() + server := httptest.NewServer(dst.handler()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + r := Registry{Address: address} + + const from = "upstream/docker.io/library/golang" + index, mine, theirs := aBaseForThisMachine(t, dst, from) + bases := []string{ + address + "/" + from + "@" + index, + // Not bases this registry holds: a package version, a tag, an image held nowhere here. + "@novox/mesh-sdk@1.2.3", + "golang:1-alpine", + "elsewhere.invalid/thing@sha256:" + strings.Repeat("e", 64), + } + n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases) + if err != nil { + t.Fatal(err) + } + // Every blob of this machine's platform is in the image's repository, so a push asks this machine's + // store for none of them; another platform's are not, since nothing built here stands on them. + for _, blob := range mine { + if !dst.links["mesh-tools-go/build"][blob] { + t.Fatalf("the base's blob %s is not in the image's repository", blob) + } + } + for _, blob := range theirs { + if dst.links["mesh-tools-go/build"][blob] { + t.Fatalf("another platform's blob %s was linked", blob) + } + } + if n != len(mine) || dst.mounts != len(mine) || dst.uploads != 0 { + t.Fatalf("%d linked, %d mounts, %d uploads; want %d mounts and no bytes moved", n, dst.mounts, dst.uploads, len(mine)) + } + + // Asked again, nothing is mounted twice. + if n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases); err != nil || n != 0 { + t.Fatalf("a second lending mounted %d: %v", n, err) + } +} + +// A registry that will not mount is said, not hidden: the push is still tried, from the store. +func TestABaseTheRegistryWillNotMountIsAnError(t *testing.T) { + dst := newRegistryOfRepositories() + dst.noMount = true + server := httptest.NewServer(dst.handler()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + index, _, _ := aBaseForThisMachine(t, dst, "upstream/thing") + _, err := Registry{Address: address}.LendBases(context.Background(), "hello/server", + []string{address + "/upstream/thing@" + index}) + if err == nil || !strings.Contains(err.Error(), "did not mount") { + t.Fatalf("a refused mount passed silently: %v", err) + } +} + +// lending records what a build lent, and when, against the push. +type lending struct { + *mirroring + order []string +} + +func (l *lending) LendBases(_ context.Context, repository string, bases []string) (int, error) { + l.order = append(l.order, "lend "+repository+" "+strings.Join(bases, " ")) + return len(bases), nil +} + +func (l *lending) PublishImage(ctx context.Context, localTag, repository string) (string, error) { + l.order = append(l.order, "push "+repository) + return l.mirroring.PublishImage(ctx, localTag, repository) +} + +func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { + r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = anImage + l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} + if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + want := []string{ + "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), + "push bus/server", + } + if strings.Join(l.order, "\n") != strings.Join(want, "\n") { + t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) + } +} diff --git a/internal/builder/reuse.go b/internal/builder/reuse.go index 04a09ce0..2def775d 100644 --- a/internal/builder/reuse.go +++ b/internal/builder/reuse.go @@ -257,6 +257,13 @@ func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest) } +func (s scratchPublisher) LendBases(ctx context.Context, repository string, bases []string) (int, error) { + if l, can := s.to.(BaseLender); can { + return l.LendBases(ctx, ScratchPrefix+repository, bases) + } + return 0, nil +} + func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) { m, can := s.to.(Mirrorer) if !can { diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 673f62f4..a186e558 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -362,6 +362,28 @@ var ControllerVerbs = []Verb{ "register": "\"true\" to register what it builds", "older": "\"true\", with register: even though a newer build of the module is registered", }, []string{"id"}, "register", "older")}, + // novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check. + {Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " + + "seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " + + "toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " + + "once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " + + "took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " + + "status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " + + "(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " + + "try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " + + "all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.", + Input: schema(map[string]string{ + "repository": "the repository on the forge, owner/name; with ref", + "ref": "the branch or commit to try, as pushed to the forge", + "base": "the branch it is merged into (the repository's default branch when none is named). The " + + "merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base", + "tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " + + "MESH_CHECK_TESTS; a script that does not runs whole", + "store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " + + "MESH_CHECK_STORE_ONLY", + "id": "instead of a repository: the id a try answered, to read what came of it", + }, nil, "store-only"), + Replaces: []string{"sh merge-check.sh", "docker run postgres"}}, // novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged. {Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " + "the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " + diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index fd86c2b9..c9f712cf 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -391,6 +392,65 @@ type PlanGate struct { // reports its module's container left running because it said busy, the judging waits and has not // started; once it reports the container replaced, Since is moved to then. Nil when nothing waited. Busy *GateBusy `json:"busy,omitempty"` + // Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that + // takes over reads the machine's last send before making it, and never sends it twice. + Pending bool `json:"pending,omitempty"` + // Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left, + // neither failed nor put back. + Silent bool `json:"silent,omitempty"` + // TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of + // the same build on the same machine, which this gate goes on from instead of starting from zero. + TookOver string `json:"took_over,omitempty"` +} + +// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR +// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found. +// A module's own count is taken only for the modules this gate judges. +func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) { + if from == nil { + return + } + if from.Since != nil { + since := *from.Since + g.Since = &since + } + for n, s := range from.Sent { + if slices.Contains(g.Machines, n) { + if g.Sent == nil { + g.Sent = map[string]SentDeclaration{} + } + g.Sent[n] = s + } + } + for n, at := range from.Applied { + if slices.Contains(g.Machines, n) { + if g.Applied == nil { + g.Applied = map[string]time.Time{} + } + g.Applied[n] = at + } + } + g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last + if from.LastPass != nil { + at := *from.LastPass + g.LastPass = &at + } + for _, m := range modules { + if n, counted := from.Healthy[m]; counted { + if g.Healthy == nil { + g.Healthy = map[string]int{} + } + g.Healthy[m] = n + } + if at, counted := from.HealthyAt[m]; counted { + if g.HealthyAt == nil { + g.HealthyAt = map[string]time.Time{} + } + g.HealthyAt[m] = at + } + } + g.Readings = append([]GateReading(nil), from.Readings...) + g.TookOver = id } // GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4). @@ -477,9 +537,17 @@ type CarriedMove struct { type PlanRelease struct { Order []string `json:"order"` Next int `json:"next"` - // Gate is the machine being judged; nil between machines. + // Gate is the first machine being judged; nil between machines. Gate *PlanGate `json:"gate,omitempty"` - Done []string `json:"done,omitempty"` + // Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by + // its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2). + Rest bool `json:"rest,omitempty"` + Gates map[string]*PlanGate `json:"gates,omitempty"` + // Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send + // within ApplySilentAfter (ADR 0353 §3 and §4). + Failed []string `json:"failed,omitempty"` + Silent []string `json:"silent,omitempty"` + Done []string `json:"done,omitempty"` // Skipped are the machines not heard from when their turn came, left as they were. Skipped []string `json:"skipped,omitempty"` // By is the person who released it, empty when the mesh did. diff --git a/internal/link/build.go b/internal/link/build.go index 6eb16588..73384e9f 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -164,6 +164,30 @@ type CheckRequest struct { // each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to // take. A build seat from before it builds nothing, and the walk builds as before. Builds []CheckBuild `json:"builds,omitempty"` + // Try marks a branch tried before its pull request (novox/hq issue 686): the repository's own check for a + // pushed branch or commit, asked by a builder through the controller's `try` verb. Its outcome is read from + // the build's log (StepTried) and said to nobody: no status on the forge, nothing recorded or registered, + // nothing built for a merge. Tests narrows what the repository's script runs to the tests a pattern names, + // and StoreOnly to its store tests; a script that reads neither runs whole. + Try bool `json:"try,omitempty"` + Tests string `json:"tests,omitempty"` + StoreOnly bool `json:"store-only,omitempty"` +} + +// StepTried is the step a build seat says a try's outcome under, in the build's log: one line, a Tried as JSON +// (novox/hq issue 686). +const StepTried = "tried" + +// Tried is what came of a branch tried before its pull request (novox/hq issue 686): the verdict of the +// repository's own check for the ref merged into the trunk, how long it took, and the lines that name what +// failed, picked from the whole of its output (issue 460). +type Tried struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + Took string `json:"took"` + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Failed []string `json:"failed,omitempty"` } // CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from diff --git a/internal/link/builds.go b/internal/link/builds.go index 7fe20541..f372d056 100644 --- a/internal/link/builds.go +++ b/internal/link/builds.go @@ -96,8 +96,27 @@ type BuildStart struct { // an earlier taker left unfinished — most often a build agent replaced mid-build — and started // again from the beginning (novox/hq issue 594). 0 from a machine that predates it. Delivery int `json:"delivery,omitempty"` + // Try marks the start of a branch tried before its pull request (novox/hq issue 686), which names no + // repository and no ref (startOf). + Try bool `json:"try,omitempty"` } +// startOf is what a build machine says of the ask it took. **A try's start names no repository and no commit** +// (novox/hq issue 686): the forge's holder writes mesh/merge-gate pending on an open pull request whose head is +// the commit a start names, and a try of that head is no check of the pull request — nothing on the forge moves +// for a try. Its id, machine and time are all the queue reads of a start. +func startOf(r BuildRequest, on string, delivery int, at time.Time) BuildStart { + s := BuildStart{ID: r.ID, Repository: r.Repository, Path: r.Path, Ref: r.Ref, On: on, + At: at.UTC().Format(time.RFC3339Nano), Delivery: delivery} + if IsTry(r) { + s.Repository, s.Path, s.Ref, s.Try = "", "", "", true + } + return s +} + +// IsTry says an ask is a branch tried before its pull request (novox/hq issue 686). +func IsTry(r BuildRequest) bool { return r.Check != nil && r.Check.Try } + // Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's // report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused // holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index c7d0c67e..5ec3b9c1 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -401,10 +401,7 @@ func (b *natsBuild) Done() error { return b.msg.Ack() } // Began publishes that this machine has taken the build, into the stream like the outcome, so a // reader that asks afterwards sees when it started as well as how it ended. func (b *natsBuild) Began(ctx context.Context) error { - body, err := json.Marshal(BuildStart{ - ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path, - Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery, - }) + body, err := json.Marshal(startOf(b.request, b.on, b.delivery, time.Now())) if err != nil { return err } diff --git a/internal/link/queue.go b/internal/link/queue.go index 50865ef8..3bbbb8a2 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -427,6 +427,9 @@ func ReadCheckBuilds(ctx context.Context, js *broker.JetStream, seat string, pul // safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern. var safeKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +// SafeBuildID says an id can be a subject token as it is: a build's id, never a pattern. +func SafeBuildID(id string) bool { return safeKey.MatchString(id) } + // Cancelled is what the controller writes for an ask it cancelled. type Cancelled struct { At string `json:"at"` diff --git a/internal/link/queue_order.go b/internal/link/queue_order.go index 449618f9..90e68f50 100644 --- a/internal/link/queue_order.go +++ b/internal/link/queue_order.go @@ -26,6 +26,8 @@ import ( // - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check // waits for ever behind a stream of deliveries; // - within a kind, first come, first served, by when it was asked; +// - a branch tried before its pull request goes after everything else, however long it has waited: it is +// never promoted ahead of a delivery, and never ahead of a pull request's check (novox/hq issue 686); // - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check // taken late breaks nothing. // @@ -64,8 +66,12 @@ func (a QueuedAsk) askedOrStored() time.Time { // QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check // that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked. func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk { - var first, then []QueuedAsk + var first, then, tries []QueuedAsk for _, a := range waiting { + if IsTry(a.Request) { + tries = append(tries, a) + continue + } if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost { then = append(then, a) continue @@ -82,7 +88,8 @@ func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk { } byAsking(first) byAsking(then) - return append(first, then...) + byAsking(tries) + return append(append(first, then...), tries...) } // toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything diff --git a/internal/link/try_test.go b/internal/link/try_test.go new file mode 100644 index 00000000..f434dc18 --- /dev/null +++ b/internal/link/try_test.go @@ -0,0 +1,55 @@ +package link + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +// The start of a try names no repository and no commit (novox/hq issue 686): the forge's holder writes +// mesh/merge-gate pending on the open pull request whose head a start names, and nothing on the forge moves +// for a try. A pull request's check still names both. +func TestATrysStartNamesNoCommitTheForgeCouldMatch(t *testing.T) { + sha := strings.Repeat("c0ffee", 7)[:40] + at := time.Date(2026, 10, 14, 0, 0, 0, 0, time.UTC) + try := BuildRequest{ID: "build-1", Repository: "https://forge/novox/a.git", Ref: sha, + Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}} + body, _ := json.Marshal(startOf(try, "ace", 1, at)) + if strings.Contains(string(body), sha) || strings.Contains(string(body), "novox/a") { + t.Fatalf("a try's start names what the forge matches a pull request by: %s", body) + } + if s := startOf(try, "ace", 2, at); s.ID != "build-1" || s.On != "ace" || !s.Try || s.Delivery != 2 || s.At == "" { + t.Fatalf("a try's start lost what the queue reads: %+v", s) + } + pull := try + pull.Check = &CheckRequest{Owner: "novox", Repo: "a", Number: 7} + if s := startOf(pull, "ace", 1, at); s.Ref != sha || s.Repository == "" || s.Try { + t.Fatalf("a pull request's check's start: %+v", s) + } +} + +// A try stays behind every delivery and every pull request's check, however long it has waited: the ten +// minutes that put a check ahead of later deliveries never promote a try (novox/hq issue 686). +func TestATryIsNeverHandedOutBeforeADeliveryOrAPullRequestsCheck(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + try := func(seq uint64, id string, asked time.Time) QueuedAsk { + a := anAsk(seq, id, asked, true) + a.Request.Check.Try = true + return a + } + waiting := []QueuedAsk{ + try(1, "try-old", now.Add(-40*time.Minute)), + try(2, "try-young", now.Add(-time.Minute)), + anAsk(3, "check-young", now.Add(-2*time.Minute), true), + anAsk(4, "delivery", now.Add(-time.Minute), false), + } + want := []string{"delivery", "check-young", "try-old", "try-young"} + if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { + t.Fatalf("handed out as %v, want %v", got, want) + } + if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") { + t.Fatal("an id that is a pattern passes for a build's id") + } +} diff --git a/merge-check.sh b/merge-check.sh index e1e85687..36de3711 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -54,11 +54,26 @@ if [ -n "$unformatted" ]; then fi CGO_ENABLED=0 go vet ./... -if command -v gcc >/dev/null 2>&1; then - CGO_ENABLED=1 go test -race -count=1 -timeout 30m ./... +suite() { + if command -v gcc >/dev/null 2>&1; then + CGO_ENABLED=1 go test -race -count=1 -timeout 30m "$@" + else + echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" + CGO_ENABLED=0 go test -count=1 -timeout 30m "$@" + fi +} +# **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb +# hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is +# handed neither and always runs whole; a part that passes is said as a part, never as the check. +if [ -n "${MESH_CHECK_TESTS:-}" ]; then + echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package" + suite -run "$MESH_CHECK_TESTS" ./... + exit 0 +fi +if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then + echo "TRIED IN PART: only the store tests listed in testdata/store-tests" else - echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" - CGO_ENABLED=0 go test -count=1 -timeout 30m ./... + suite ./... fi # **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip diff --git a/module.json b/module.json index 1b84a8c4..538f5cca 100644 --- a/module.json +++ b/module.json @@ -77,6 +77,7 @@ "replay", "kill", "forget-checks", + "try", "pause", "resume", "hand-act", diff --git a/testdata/store-tests b/testdata/store-tests index d3af9659..62ca1239 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -64,3 +64,14 @@ ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves +./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses +./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn +./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere +./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain +./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain +./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk +./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep +./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid +./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport +./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition +./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn