From 1af40279b122b0ff82ef2f4f05c2719b6f1de5eb Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 16:33:29 +0100 Subject: [PATCH] =?UTF-8?q?Builder:=20its=20candidates,=20modules,=20manif?= =?UTF-8?q?ests,=20sources,=20steps=20and=20toolchains=20are=20the=20catal?= =?UTF-8?q?og's,=20the=20gate's=20and=20the=20build=20seats'=20types=20and?= =?UTF-8?q?=20the=20clients'=20codecs,=20its=20build=20ask=20the=20Deliver?= =?UTF-8?q?y=20SDK's,=20its=20artifacts=20the=20forms'=20type;=20the=20hol?= =?UTF-8?q?ders=20and=20buckets=20it=20names=20are=20its=20own=20constants?= =?UTF-8?q?=20checked=20against=20its=20manifest;=20no=20go/delivery=20lef?= =?UTF-8?q?t=20(D-11;=20ADR=200634=20=C2=A71,=20=C2=A72,=20=C2=A76;=20issu?= =?UTF-8?q?e=201966)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-delivery-builder/buildrun/go.mod | 1 - .../mesh-delivery-builder/builds/builds.go | 41 +-- .../cmd/mesh-delivery-builder-tools/main.go | 11 +- .../cmd/mesh-delivery-builder/branchend.go | 12 +- .../cmd/mesh-delivery-builder/bus.go | 8 +- .../cmd/mesh-delivery-builder/findings.go | 16 +- .../gone_integration_test.go | 20 +- .../cmd/mesh-delivery-builder/held.go | 43 +-- .../mesh-delivery-builder/integration_test.go | 245 +++++++-------- .../cmd/mesh-delivery-builder/main.go | 55 ++-- .../cmd/mesh-delivery-builder/names.go | 38 +++ .../cmd/mesh-delivery-builder/names_test.go | 50 ++++ .../cmd/mesh-delivery-builder/oldform.go | 8 +- .../cmd/mesh-delivery-builder/optional.go | 12 +- .../cmd/mesh-delivery-builder/reads.go | 4 +- .../cmd/mesh-delivery-builder/reasks.go | 12 +- .../cmd/mesh-delivery-builder/refused_test.go | 51 ++-- .../cmd/mesh-delivery-builder/seats.go | 8 +- .../cmd/mesh-delivery-builder/tagbounds.go | 6 +- .../cmd/mesh-delivery-builder/toolchains.go | 4 +- .../cmd/mesh-delivery-builder/versions.go | 18 +- modules/mesh-delivery-builder/go.mod | 21 +- .../internal/build/announce.go | 10 +- .../internal/build/announce_test.go | 10 +- .../internal/build/askagain.go | 21 +- .../internal/build/askfake_test.go | 27 +- .../internal/build/buildagain.go | 22 +- .../internal/build/candidates.go | 32 +- .../internal/build/candidates_test.go | 34 ++- .../internal/build/claim.go | 51 ++-- .../internal/build/claim_test.go | 38 +-- .../internal/build/fakes_test.go | 23 +- .../internal/build/fixtures_test.go | 17 +- .../internal/build/followups_test.go | 22 +- .../internal/build/gone_test.go | 24 +- .../internal/build/imports_test.go | 19 +- .../internal/build/numbers.go | 40 +-- .../internal/build/plan.go | 10 +- .../internal/build/plan_test.go | 12 +- .../internal/build/queue.go | 7 +- .../internal/build/r4e_test.go | 95 +++--- .../internal/build/round4_test.go | 20 +- .../internal/build/run.go | 216 +++++++------- .../internal/build/run_test.go | 282 +++++++++--------- .../internal/build/since.go | 18 +- .../internal/build/stuck.go | 27 +- .../internal/build/toolchains.go | 28 ++ .../internal/versions/again.go | 4 +- .../internal/versions/claim.go | 4 +- .../internal/versions/names.go | 16 +- .../internal/versions/oldform.go | 13 +- .../internal/versions/tools.go | 10 +- .../internal/versions/versions_test.go | 16 +- 53 files changed, 1007 insertions(+), 845 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/toolchains.go diff --git a/modules/mesh-delivery-builder/buildrun/go.mod b/modules/mesh-delivery-builder/buildrun/go.mod index ede9fa15..9cb38ea8 100644 --- a/modules/mesh-delivery-builder/buildrun/go.mod +++ b/modules/mesh-delivery-builder/buildrun/go.mod @@ -4,7 +4,6 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index b52b676d..edabc9d9 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-delivery/sdk/forms" "git.novox.be/novox/mesh-delivery/sdk/kept" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" @@ -90,39 +91,15 @@ func ParseBuilderBuildKey(key string) (module, fp string, err error) { return module, fp, nil } -// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and -// 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision -// L1 of R4e), by kind: -// -// bundle, archive artifact-store:////blobs/ the blob pushed to the artifact store -// image, upstream //@ the image registry's reference, which -// artifact-store:///@ catalogue.Recorded keeps without its address -// package @ the version published from the trunk -// -// From the registry milestone a done key names the reference its seat answered (novox/hq ADR 0605 §9, design 51 *The -// done key names the reference the seat answered*): `//@`, the registry seat it -// came through (oci-artifact-registry for a bundle or an archive, container-image-registry for an image or an upstream -// copy), and `@` for a package as before. A key written in the older form keeps it; this reads both, and -// no build is built again for its form (seats.InArtifactStore reads both the same way). -// -// is Name and is Digest. A reference is empty for a candidate's image or package, which is never -// pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose -// every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked. -type BuiltArtifact struct { - Name string `json:"name"` - Kind string `json:"kind,omitempty"` - Digest string `json:"digest"` - Reference string `json:"reference,omitempty"` - // References are every form this artifact is named in, Reference among them (novox/hq ADR 0618, ADR 0624): a build - // keeps each form that a counted node-engine states, away nodes included, and Reference stays the form every one - // of them reads. A reader that does not know the field reads Reference alone. - References []string `json:"references,omitempty"` -} +// BuiltArtifact is one artifact of a done build: its name, kind, digest and the reference that names where it is, with every +// form it is named in (novox/hq ADR 0618, ADR 0624). The Delivery SDK's forms package holds the type, since the forms read it +// and forms cannot import this package. +type BuiltArtifact = forms.BuiltArtifact const artifactStore = "artifact-store://" // referenceProblem says what is wrong with a's reference for its kind, or "". -func (a BuiltArtifact) referenceProblem() string { +func referenceProblem(a BuiltArtifact) string { r := a.Reference if r == "" || a.Kind == "" { return "" @@ -284,14 +261,14 @@ func (b BuilderBuild) check() error { if a.Digest == "" { return fmt.Errorf("builds: the artifact %q of the build has no digest", a.Name) } - if p := a.referenceProblem(); p != "" { + if p := referenceProblem(a); p != "" { return fmt.Errorf("builds: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p) } if len(a.References) > 0 && !slices.Contains(a.References, a.Reference) { return fmt.Errorf("builds: the references of the %s %q do not name its reference %q", a.Kind, a.Name, a.Reference) } for _, r := range a.References { - if p := (BuiltArtifact{Name: a.Name, Kind: a.Kind, Digest: a.Digest, Reference: r}).referenceProblem(); p != "" { + if p := referenceProblem(BuiltArtifact{Name: a.Name, Kind: a.Kind, Digest: a.Digest, Reference: r}); p != "" { return fmt.Errorf("builds: the reference %q of the %s %q %s", r, a.Kind, a.Name, p) } } @@ -319,7 +296,7 @@ func (b BuilderBuild) check() error { if b.State != BuildQueued { return fmt.Errorf("builds: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits) } - if p := (BuiltArtifact{Kind: "package", Reference: b.Waits}).referenceProblem(); p != "" { + if p := referenceProblem(BuiltArtifact{Kind: "package", Reference: b.Waits}); p != "" { return fmt.Errorf("builds: the package a key waits for, %q, %s", b.Waits, p) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 1858af75..acb6b77c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -16,8 +16,9 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-delivery/sdk/enginereads" @@ -192,7 +193,7 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer, "next": build.StuckTool + " shows the key and its asks while they wait"}, nil }}, - {Name: delivery.BuildsSinceTool, + {Name: builds.BuildsSinceTool, Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " + "fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + "queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " + @@ -250,7 +251,7 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q language = build.LanguageOf(str(args, "seat")) } if language == "" { - language = delivery.AskGo + language = buildstep.AskGo } limit, _ := args["limit"].(float64) key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now()) @@ -406,7 +407,7 @@ func main() { } return e.Value, true, nil }} - tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now, vr, tags, tr, fr, ofr) + tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, tags, tr, fr, ofr) // The builder's expiring keys (state bounds): what each bounds and when it ends. tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription, Input: map[string]any{"type": "object", "properties": map[string]any{}}, @@ -419,7 +420,7 @@ func main() { tools = append(tools, enginereads.Tool(EngineReadsTool, "the builder", enginereads.KeptState(stdio.State("engine-reads")), true, time.Now)) // builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's // states stand, and asking one again by hand (ADR 0615, DLV-M1). - tools = append(tools, views.StatusTools(delivery.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + tools = append(tools, views.StatusTools(builds.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go index a63c2785..bcfc6256 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go @@ -9,7 +9,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/health" "mesh-delivery-builder/internal/build" @@ -20,14 +20,14 @@ import ( // branch's end (design 51 *Tagged versions*). const forgeStream = "FORGE" -// forgeBranchDeleted is the forge bridge's kind for a branch's end (delivery.ForgeBranchDeleted in the group's mesh-sdk). +// forgeBranchDeleted is the forge bridge's kind for a branch's end (forgeBranchDeleted in the group's mesh-sdk). const forgeBranchDeleted = "branch-deleted" // checkerAsksStream is the checker's family `asks` as the bus names it, and tagWantedSubject its event tag-wanted // (design 51 *Tagged versions*, DLV-C3): on the checker's own stream, so its subject is mesh.stream..event.*. const ( checkerAsksStream = "ASKS" - tagWantedSubject = "mesh.stream." + delivery.CheckerModule + ".event.tag-wanted" + tagWantedSubject = "mesh.stream." + checkerModule + ".event.tag-wanted" ) // forgeEnd is the part of a forge message the builder reads. @@ -41,10 +41,10 @@ type forgeEnd struct { // ended says whether the message ends a branch: merged, a pull closed, or branch-deleted. func (f forgeEnd) ended() bool { switch f.Kind { - case delivery.ForgeMerged, forgeBranchDeleted: + case forgemsg.ForgeMerged, forgeBranchDeleted: return f.Branch != "" - case delivery.ForgePull: - return f.Action == delivery.PullClosed && f.Branch != "" + case forgemsg.ForgePull: + return f.Action == forgemsg.PullClosed && f.Branch != "" } return false } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go index 4023faf2..34654335 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go @@ -7,8 +7,8 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-builder/internal/build" ) @@ -111,7 +111,7 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key type manifestReader struct{ kv jetstream.KeyValue } func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) { - key, err := delivery.ManifestsKey(digest) + key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { return catalogue.Manifest{}, err } @@ -119,10 +119,10 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue. if err != nil { return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err) } - kept, err := delivery.DecodeManifest(e.Value()) + kept, err := meshcatalogclient.DecodeManifest(e.Value()) if err != nil { return catalogue.Manifest{}, err } // Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside. - return catalogue.Decode([]byte(kept.JSON)) + return catalogue.Decode([]byte(kept.Manifest)) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go index 6c458d2c..4d2e8a42 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/findings.go @@ -7,7 +7,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" @@ -18,7 +18,7 @@ import ( // findingsBucket is the builder's findings (novox/hq ADR 0564 §2): what it sees wrong now, written at its raise and // deleted at its clear, which the condition keeper raises and clears. -var findingsBucket = health.FindingsBucket(delivery.BuilderModule) +var findingsBucket = health.FindingsBucket(builds.BuilderModule) // findings writes the builder's findings into its state, once bound; until then each is said in the log only. type findings struct { @@ -63,19 +63,19 @@ const BuildFailed = "build-failed" // buildEnded raises the module's build-failed for a trunk build that failed and clears it at a later done one; a // candidate's build (its ref named) is the check's to say. -func (f *findings) buildEnded(ctx context.Context, e delivery.BuildLogEntry) { - if e.Event != delivery.BuildEndedEvent || e.Ref != "" { +func (f *findings) buildEnded(ctx context.Context, e builds.BuildLogEntry) { + if e.Event != builds.BuildEndedEvent || e.Ref != "" { return } switch e.Outcome { - case delivery.BuildFailed: + case builds.BuildFailed: f.raise(ctx, finding{kind: BuildFailed, scope: health.ScopeModule, id: e.Module, summary: "the trunk build of " + e.Module + " at " + e.FP + " (commit " + e.Commit + ") failed", headline: "A module's trunk build failed", explanation: "The builder built " + e.Module + " from the trunk and the build failed, so its new version is not delivered.", resolved: "A later trunk build of " + e.Module + " is done.", needs: "builder_builds_since shows the build; builder_build_again builds it again."}) - case delivery.BuildDone: + case builds.BuildDone: f.clear(ctx, BuildFailed, health.ScopeModule, e.Module) } } @@ -100,7 +100,7 @@ func (f *findings) raise(ctx context.Context, fd finding) { } value, err := health.EncodeFinding(health.Finding{Key: key, Kind: fd.kind, Subject: health.Subject{Scope: fd.scope, ID: fd.id}, Severity: sev, Summary: fd.summary, Headline: fd.headline, Explanation: fd.explanation, Resolved: fd.resolved, Needs: fd.needs, - Evidence: []health.Evidence{{At: at, Said: fd.summary}}, Source: "builder." + fd.kind, Observer: delivery.BuilderModule, At: at}) + Evidence: []health.Evidence{{At: at, Said: fd.summary}}, Source: "builder." + fd.kind, Observer: builds.BuilderModule, At: at}) if err == nil { _, err = kv.Put(ctx, key, value) } @@ -135,7 +135,7 @@ func (f *findings) notHeard(ctx context.Context) heard.Writer { f.kv.Ready(func(kv jetstream.KeyValue) { do(kv) }) } } - return heard.Writer{Observer: delivery.BuilderModule, Logf: f.logf, Now: f.now, + return heard.Writer{Observer: builds.BuilderModule, Logf: f.logf, Now: f.now, Put: func(key string, value []byte) error { on(func(kv findingStore) { if _, err := kv.Put(ctx, key, value); err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go index ee342ee6..52be8bde 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go @@ -13,8 +13,8 @@ import ( "github.com/nats-io/nats.go/jetstream" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // A done key whose blob the artifact store no longer holds is built again, by the builder whole against the bus @@ -78,10 +78,10 @@ func eventsConsumer(t *testing.T, b *bed, node string) { // doneKey writes builds.. done, naming one archive at reference, and answers its revision. func doneKey(t *testing.T, b *bed, module, manifest, reference, digest string) uint64 { t.Helper() - key, _ := delivery.BuilderBuildKey(module, fp) - data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "n2.mesh-delivery-builder", + key, _ := builds.BuilderBuildKey(module, fp) + data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "n2.mesh-delivery-builder", Manifest: manifest, Step: "built mesh-build-go on n2", - Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest, Reference: reference}}}) + Artifacts: []builds.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest, Reference: reference}}}) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func doneKey(t *testing.T, b *bed, module, manifest, reference, digest string) u func revisionOf(t *testing.T, b *bed, module string) uint64 { t.Helper() - key, _ := delivery.BuilderBuildKey(module, fp) + key, _ := builds.BuilderBuildKey(module, fp) e, err := b.builds.Get(b.ctx, key) if err != nil { t.Fatal(err) @@ -115,8 +115,8 @@ func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) { h.serve(t, b) b.start(t, "n1", nil) - v := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool { - return v.State == delivery.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Digest == digestOf("files") + v := b.await(t, "ca-trust", func(v builds.BuilderBuild) bool { + return v.State == builds.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Digest == digestOf("files") }) if !strings.Contains(v.Step, "built again: the artifact store no longer held "+oldRef) { t.Errorf("the rebuilt key's step %q does not say why it was built again", v.Step) @@ -168,7 +168,7 @@ func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) { mu.Lock() collected = true mu.Unlock() - key, _ := delivery.BuilderBuildKey("ca-trust", fp) + key, _ := builds.BuilderBuildKey("ca-trust", fp) at := time.Now().UTC() body, err := json.Marshal(sweep.ArtifactsGone{Event: sweep.EventArtifactsGone, Module: "ca-trust", Key: key, Fingerprint: fp, Gone: []string{ref}, At: &at}) @@ -178,8 +178,8 @@ func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) { if _, err := b.js.Publish(b.ctx, sweep.SubjectArtifactsGone, body); err != nil { t.Fatal(err) } - v := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool { - return v.State == delivery.BuildDone && strings.Contains(v.Step, "built again") + v := b.await(t, "ca-trust", func(v builds.BuilderBuild) bool { + return v.State == builds.BuildDone && strings.Contains(v.Step, "built again") }) if got := revisionOf(t, b, "ca-trust"); got <= rev || b.fake.count("mesh-build-go.build_go_build") != 1 { t.Errorf("build = %+v at revision %d (was %d), %d builds: want built again once on the event", v, got, rev, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go index 369a245e..acdbd9fe 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -10,7 +10,8 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" "mesh-delivery-builder/internal/build" ) @@ -22,7 +23,7 @@ type heldOnTheBus struct{ js jetstream.JetStream } // Held is build.Builder's Held: a step key `.` in the build module's steps, or a message of the stream under // the ask's subject whose Nats-Msg-Id is the ask's id. -func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) { +func (h heldOnTheBus) Held(ctx context.Context, a buildstep.StepAsk) (bool, error) { seen, err := h.seen(ctx, a, false) if err != nil { return false, err @@ -31,7 +32,7 @@ func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error } // Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "". -func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) { +func (h heldOnTheBus) Gone(ctx context.Context, a buildstep.StepAsk) (string, error) { seen, err := h.seen(ctx, a, true) if err != nil { return "", err @@ -48,17 +49,17 @@ const ( // seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when // the ask is on its stream. -func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) { +func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool) (build.AskSeen, error) { seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)} - module, err := delivery.BuildModuleOf(a.Language) + module, err := buildstep.BuildModuleOf(a.Language) if err != nil { return seen, err } - steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) + steps, err := h.js.KeyValue(ctx, buildrun.StepsBucket(module)) if err != nil { - return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + return seen, fmt.Errorf("the bucket %s: %w", buildrun.StepsBucket(module), err) } - pattern, err := delivery.AskStepWatch(a.ID) + pattern, err := buildstep.AskStepWatch(a.ID) if err != nil { return seen, err } @@ -114,13 +115,13 @@ func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) // stream is the build seat's stream of language, its name and the subject of kind on it. func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) { - subject, err := delivery.AskSubject(language, kind) + subject, err := buildstep.AskSubject(language, kind) if err != nil { return nil, "", "", err } - // The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its + // The seat's stream by its name (buildstep.AskStream), as the broker grants the builder its information and its // messages by sequence and nothing wider (issue 1910). - name, err := delivery.AskStream(language) + name, err := buildstep.AskStream(language) if err != nil { return nil, "", "", err } @@ -145,7 +146,7 @@ func onSubject(ctx context.Context, stream jetstream.Stream, subject string, eac return err } q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time} - if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil { + if _, kind, err := buildstep.ParseAskSubject(m.Subject); err == nil { q.Kind = kind } if q.At = build.AskedAt(q.Ask); q.At.IsZero() { @@ -185,7 +186,7 @@ func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) str if err != nil { continue } - if s, err := delivery.DecodeStep(e.Value()); err == nil { + if s, err := buildstep.DecodeStep(e.Value()); err == nil { return s.State } } @@ -214,7 +215,7 @@ func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][ if err != nil { continue } - s, err := delivery.DecodeStep(e.Value()) + s, err := buildstep.DecodeStep(e.Value()) if err != nil { continue } @@ -230,7 +231,7 @@ func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][ // Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its // subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.. -func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error { +func (h heldOnTheBus) Drop(ctx context.Context, a buildstep.StepAsk) error { stream, name, subject, err := h.stream(ctx, a.Language, a.Kind) if errors.Is(err, jetstream.ErrStreamNotFound) { return nil @@ -258,11 +259,11 @@ func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error { // every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits. func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) { q := build.SeatQueue{Language: language, Note: build.QueueNote} - module, err := delivery.BuildModuleOf(language) + module, err := buildstep.BuildModuleOf(language) if err != nil { return q, err } - stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery) + stream, name, _, err := h.stream(ctx, language, buildstep.AskDelivery) q.Stream = name if errors.Is(err, jetstream.ErrStreamNotFound) { q.Note = name + " does not exist" @@ -276,16 +277,16 @@ func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (bu return q, err } q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq - steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) + steps, err := h.js.KeyValue(ctx, buildrun.StepsBucket(module)) if err != nil { - return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + return q, fmt.Errorf("the bucket %s: %w", buildrun.StepsBucket(module), err) } taken, err := stepKeys(ctx, steps) if err != nil { return q, err } - for _, kind := range delivery.AskKinds { - subject, _ := delivery.AskSubject(language, kind) + for _, kind := range buildstep.AskKinds { + subject, _ := buildstep.AskSubject(language, kind) var ahead []build.Queued if err := onSubject(ctx, stream, subject, func(m build.Queued) bool { ahead = append(ahead, m) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 882a56a8..92200937 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -26,13 +26,20 @@ import ( // The build runner the build modules pull and run asks with: mesh-build-go's own package (novox/hq ADR 0593 §1), // another repository's, by its published version, so the builder is tested against the real pulling code. runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" - "git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry" - "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry" - sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" - "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views/viewstest" @@ -139,7 +146,7 @@ func (f *fakeBuildModules) step(module, name string, args map[string]any) (any, d := digestOf(fmt.Sprint(args["artifact"])) return json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`), nil case tool != "build_image_build" && f.refusePush.Load(): - return nil, fmt.Errorf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"]) + return nil, fmt.Errorf("%s%v: the artifact store answered 507 Insufficient Storage", builds.NotStored, args["artifact"]) case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1) d := digestOf(fmt.Sprint(args["artifact"])) return json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://` + filepath.Base(fmt.Sprint(args["module"])) + `/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`), nil @@ -153,7 +160,7 @@ func (f *fakeBuildModules) step(module, name string, args map[string]any) (any, // serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on nodes. func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { - kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1}) + kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: buildrun.StepsBucket(module), History: 1}) if err != nil { t.Fatal(err) } @@ -177,7 +184,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { } return 2 }, PullWait: 200 * time.Millisecond, - Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }} + Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }} f.mu.Lock() f.pullers = append(f.pullers, p) f.mu.Unlock() @@ -191,11 +198,11 @@ const trunkHead = "3333333333333333333333333333333333333333" // trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision. func (b *bed) trunk(t *testing.T, head string) uint64 { t.Helper() - data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}}) + data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{V: keys.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}}) if err != nil { t.Fatal(err) } - rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data) + rev, err := b.source.Put(b.ctx, meshdeliveryclient.SourceMainKey, data) if err != nil { t.Fatal(err) } @@ -273,7 +280,7 @@ func startBedWith(t *testing.T, closed bool) *bed { t.Fatal(err) } } - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket(module), History: 1}) + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buildrun.ToolchainsBucket(module), History: 1}) if err != nil { t.Fatal(err) } @@ -286,19 +293,19 @@ func startBedWith(t *testing.T, closed bool) *bed { } return kv } - b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory) - b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory) - b.source = mk(delivery.SourceBucket, delivery.SourceHistory) - mk(delivery.ManifestsBucket, delivery.ManifestsHistory) + b.modules = mk(indexerBucket(modulesState), modulesHistory) + b.candidates = mk(indexerBucket(candidatesState), candidatesHistory) + b.source = mk(sourceBucket, sourceHistory) + mk(indexerBucket(manifestsState), manifestsHistory) // The indexer and the assigner as the owners of the states the builder follows: their read tools and views families // (ADR 0615, DLV-M1); n1 and n2 each have their durable consumer. - readers := []string{"n1_" + delivery.BuilderModule, "n2_" + delivery.BuilderModule} + readers := []string{"n1_" + builds.BuilderModule, "n2_" + builds.BuilderModule} for _, st := range []string{modulesState, manifestsState, candidatesState} { - viewstest.Serve(t, ctx, js, ctl, delivery.IndexerModule, st, delivery.IndexerModule+".indexer_read_"+st, readers...) + viewstest.Serve(t, ctx, js, ctl, indexerModule, st, indexerModule+".indexer_read_"+st, readers...) } - b.packages = viewstest.Serve(t, ctx, js, ctl, delivery.IndexerModule, delivery.PackagesState, delivery.IndexerModule+".indexer_read_packages", readers...) - viewstest.Serve(t, ctx, js, ctl, delivery.AssignerModule, delivery.PlacementState, delivery.AssignerModule+".assigner_read_placement", readers...) - b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) + b.packages = viewstest.Serve(t, ctx, js, ctl, indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages", readers...) + viewstest.Serve(t, ctx, js, ctl, assignerModule, "placement", assignerModule+".assigner_read_placement", readers...) + b.builds = mk(builds.BuilderBuildsBucket, builds.BuilderBuildsHistory) b.instances = mk(instancesBucket, 1) // The tag asks and their bounds (design 51 *Tagged versions*): the bounds declared key-ttl, so an ended bound leaves the // bus's marker a watch hears. @@ -308,8 +315,8 @@ func startBedWith(t *testing.T, closed bool) *bed { } b.out = &saidLog{} // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. - logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog) - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, + logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog) + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } @@ -326,20 +333,20 @@ func startBedWith(t *testing.T, closed bool) *bed { } // logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user. -func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry { +func (b *bed) logged(t *testing.T, module string) []builds.BuildLogEntry { t.Helper() conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"}) if err != nil { t.Fatal(err) } defer conn.Close() - entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module) + entries, err := conn.ReadLog(builds.BuilderModule, builds.BuilderLog).Since(time.Unix(0, 0), module) if err != nil { t.Fatal(err) } - var out []delivery.BuildLogEntry + var out []builds.BuildLogEntry for _, le := range entries { - e, err := delivery.DecodeBuildLogEntry(le.Value) + e, err := builds.DecodeBuildLogEntry(le.Value) if err != nil { t.Fatal(err) } @@ -351,11 +358,11 @@ func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry { // report writes a node's toolchains key of a build module and says so on the module's stream, as the module does. func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { t.Helper() - data, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()}) + data, err := sdkstep.EncodeToolchain(sdkstep.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()}) if err != nil { t.Fatal(err) } - key, _ := delivery.ToolchainKey(node, language) + key, _ := sdkstep.ToolchainKey(node, language) if _, err := b.tools[module].Put(b.ctx, key, data); err != nil { t.Fatal(err) } @@ -370,39 +377,39 @@ func (b *bed) module(t *testing.T, name, fixture, path string) string { } // bothToolchains are the toolchains the bed's nodes report, as a module's fingerprint names them. -var bothToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} +var bothToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} // moduleWith is module with the toolchains its fingerprint was computed with. -func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed) string { +func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed) string { return b.moduleCommit(t, name, fixture, path, toolchains, commit) } -func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at string) string { +func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed, at string) string { t.Helper() return b.moduleFP(t, name, fixture, path, toolchains, at, fp, 0) } // moduleFP is moduleCommit with the new-form fingerprint fp2 and the source revision src of its merge (0: none). -func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at, fp2 string, src uint64) string { +func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed, at, fp2 string, src uint64) string { t.Helper() raw, err := os.ReadFile(filepath.Join("testdata", fixture)) if err != nil { t.Fatal(err) } - digest := delivery.ManifestDigest(raw) - manifests, err := b.js.KeyValue(b.ctx, delivery.ManifestsBucket) + digest := meshcatalogclient.ManifestDigest(raw) + manifests, err := b.js.KeyValue(b.ctx, indexerBucket(manifestsState)) if err != nil { t.Fatal(err) } - mv, err := delivery.EncodeManifest(delivery.Manifest{Module: name, JSON: string(raw)}) + mv, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: name, Manifest: string(raw)}) if err != nil { t.Fatal(err) } if _, err := manifests.Put(b.ctx, digest, mv); err != nil { t.Fatal(err) } - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: at, FP: "src1:" + strings.Repeat("f", 64), FP2: fp2, - Manifest: digest, Path: path, Toolchains: toolchains, Src: src}) + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: at, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2, + Manifest: digest, Path: path, Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)}) if err != nil { t.Fatal(err) } @@ -476,25 +483,25 @@ func (b *bed) awaitSaid(t *testing.T, want string) string { } // await reads the build key until pred holds of it, within 20 s. -func (b *bed) await(t *testing.T, module string, pred func(delivery.BuilderBuild) bool) delivery.BuilderBuild { +func (b *bed) await(t *testing.T, module string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { t.Helper() return b.awaitFP(t, module, fp, pred) } // awaitFP is await of the build key of fingerprint at. -func (b *bed) awaitFP(t *testing.T, module, at string, pred func(delivery.BuilderBuild) bool) delivery.BuilderBuild { +func (b *bed) awaitFP(t *testing.T, module, at string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { t.Helper() - key, err := delivery.BuilderBuildKey(module, at) + key, err := builds.BuilderBuildKey(module, at) if err != nil { t.Fatal(err) } - var last delivery.BuilderBuild + var last builds.BuilderBuild for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { e, err := b.builds.Get(b.ctx, key) if err != nil { continue } - if last, err = delivery.DecodeBuilderBuild(e.Value()); err == nil && pred(last) { + if last, err = builds.DecodeBuilderBuild(e.Value()); err == nil && pred(last) { return last } } @@ -502,8 +509,8 @@ func (b *bed) awaitFP(t *testing.T, module, at string, pred func(delivery.Builde return last } -func isState(state string) func(delivery.BuilderBuild) bool { - return func(v delivery.BuilderBuild) bool { return v.State == state } +func isState(state string) func(builds.BuilderBuild) bool { + return func(v builds.BuilderBuild) bool { return v.State == state } } // TestBuilderMakesADoneKeyWithTheDigests: modules.netcheck (a Go and a TypeScript bundle) and its manifest are in the @@ -529,7 +536,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Fatal(err) } b.start(t, "n1", map[string]string{"MESH_BUILDER_TAG_ASK_BOUND": "1s", "MESH_BUILDER_STEP_BOUND": "1s"}) - v := b.await(t, "netcheck", isState(delivery.BuildDone)) + v := b.await(t, "netcheck", isState(builds.BuildDone)) for _, a := range []string{"tools-go", "tools-typescript"} { es := reg.of(t, "netcheck/"+a) if len(es) != 1 || es[0].Version != "0.1.5" || es[0].Fingerprint != fp || es[0].State != versions.Numbered || es[0].Digest != digestOf(a) { @@ -554,15 +561,15 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 // 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a // reader with the log's read reads them, the end naming the manifest and what was made. // DLV-B10: between them the build log holds the build's narrations (build.said), each version numbered, and why. - var ended []delivery.BuildLogEntry + var ended []builds.BuildLogEntry for _, e := range b.logged(t, "netcheck") { - if e.Event != delivery.BuildSaidEvent { + if e.Event != builds.BuildSaidEvent { ended = append(ended, e) } else if e.Why != versions.SaidNumbered || !strings.Contains(e.Line, "numbered 0.1.5") || e.FP != fp { t.Errorf("said = %+v, want netcheck's versions numbered", e) } } - if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent || + if len(ended) != 2 || ended[0].Event != builds.BuildStartedEvent || ended[1].Event != builds.BuildEndedEvent || ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" { t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended) } @@ -576,11 +583,11 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Errorf("the narration = %q, want what the ask waits on", said) } // And in the build log, under the builder's own key (the ruling of 13:17: build.said). - if !slices.ContainsFunc(b.logged(t, delivery.BuilderModule), func(e delivery.BuildLogEntry) bool { - return e.Event == delivery.BuildSaidEvent && e.Why == versions.SaidUnanswered && e.FP == key && e.Commit == "abc" && + if !slices.ContainsFunc(b.logged(t, builds.BuilderModule), func(e builds.BuildLogEntry) bool { + return e.Event == builds.BuildSaidEvent && e.Why == versions.SaidUnanswered && e.FP == key && e.Commit == "abc" && strings.Contains(e.Line, "the check's build of abc is not claimed yet") }) { - t.Errorf("the build log of %s = %+v, want the unanswered tag ask said", delivery.BuilderModule, b.logged(t, delivery.BuilderModule)) + t.Errorf("the build log of %s = %+v, want the unanswered tag ask said", builds.BuilderModule, b.logged(t, builds.BuilderModule)) } if e, err := b.tagAsks.Get(b.ctx, key); err != nil { t.Fatal(err) @@ -647,7 +654,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b.fake.hang.Store(&release) third := "src2:" + strings.Repeat("4", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) - if f := b.awaitFP(t, "netcheck", third, isState(delivery.BuildFailed)); !strings.Contains(f.Error, "did not end within") { + if f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)); !strings.Contains(f.Error, "did not end within") { t.Errorf("the failed build = %+v, want its bound run out", f) } failed := health.FindingKey(health.ScopeModule, "netcheck", BuildFailed) @@ -655,10 +662,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b.fake.hang.Store(nil) close(release) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) // the indexer writes it again - b.awaitFP(t, "netcheck", third, isState(delivery.BuildDone)) + b.awaitFP(t, "netcheck", third, isState(builds.BuildDone)) b.awaitSaid(t, "kept: fingerprint "+third+" is numbered under it") - if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e delivery.BuildLogEntry) bool { - return e.Event == delivery.BuildSaidEvent && e.Why == versions.SaidKept && e.FP == third && strings.Contains(e.Line, "kept: fingerprint "+third) + if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e builds.BuildLogEntry) bool { + return e.Event == builds.BuildSaidEvent && e.Why == versions.SaidKept && e.FP == third && strings.Contains(e.Line, "kept: fingerprint "+third) }) { t.Errorf("the build log of netcheck says no version kept at %s", third) } @@ -685,8 +692,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if _, err := reg.versions.Put(b.ctx, versions.Key(tagged.Name, tagged.Version), raw); err != nil { t.Fatal(err) } - if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(delivery.ForgeBridgeModule, "forge"), - []byte(`{"kind":"`+delivery.ForgeMerged+`","repository":"novox/mesh-sdk","branch":"feat/x"}`)); err != nil { + if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(forgeBridgeModule, "forge"), + []byte(`{"kind":"`+forgemsg.ForgeMerged+`","repository":"novox/mesh-sdk","branch":"feat/x"}`)); err != nil { t.Fatal(err) } for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) { @@ -715,7 +722,7 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) { for _, c := range []struct { name, path string two bool - }{{"no toolchain", "modules/ca-trust", false}, {"root", delivery.RootPath, false}, {"two instances together", "modules/ca-trust", true}} { + }{{"no toolchain", "modules/ca-trust", false}, {"root", meshcatalogclient.RootPath, false}, {"two instances together", "modules/ca-trust", true}} { t.Run(c.name, func(t *testing.T) { b := startBed(t) b.archive(t, "ca-trust", c.path, commit) @@ -723,7 +730,7 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) { if c.two { b.start(t, "n2", nil) } - v := b.await(t, "ca-trust", isState(delivery.BuildDone)) + v := b.await(t, "ca-trust", isState(builds.BuildDone)) if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) { t.Errorf("build = %+v, want done naming no toolchain", v) } @@ -745,11 +752,11 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFing b.fake.failBuild.Store(true) b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) - v := b.await(t, "ca-trust", isState(delivery.BuildFailed)) + v := b.await(t, "ca-trust", isState(builds.BuildFailed)) if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") { t.Errorf("build = %+v, want failed naming the node and the step's reason", v) } - key, _ := delivery.BuilderBuildKey("ca-trust", fp) + key, _ := builds.BuilderBuildKey("ca-trust", fp) failed, err := b.builds.Get(b.ctx, key) if err != nil { t.Fatal(err) @@ -757,7 +764,7 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFing b.fake.failBuild.Store(false) // The indexer writes the module again (here at a later commit), with the same fingerprint. b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef") - b.await(t, "ca-trust", isState(delivery.BuildDone)) + b.await(t, "ca-trust", isState(builds.BuildDone)) done, err := b.builds.Get(b.ctx, key) if err != nil { t.Fatal(err) @@ -778,8 +785,8 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A b.start(t, "n1", nil) // Another module, whose fingerprint names the new toolchain, is built; by then the old one has been looked at. b.moduleCommit(t, "netcheck2", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck", - map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit) - b.await(t, "netcheck2", isState(delivery.BuildDone)) + map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit) + b.await(t, "netcheck2", isState(builds.BuildDone)) if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil { t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2") } @@ -789,7 +796,7 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A } func mustKey(t *testing.T, module string) string { - k, err := delivery.BuilderBuildKey(module, fp) + k, err := builds.BuilderBuildKey(module, fp) if err != nil { t.Fatal(err) } @@ -799,14 +806,14 @@ func mustKey(t *testing.T, module string) string { // moduleAt is module with the go toolchain version the fingerprint names and the commit it was computed at. func (b *bed) moduleAt(t *testing.T, name, fixture, path, goVersion, at string) { t.Helper() - b.moduleCommit(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: goVersion, Recipe: "1"}}, at) + b.moduleCommit(t, name, fixture, path, map[string]builds.ToolchainUsed{"go": {Version: goVersion, Recipe: "1"}}, at) } // TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart: the builds bucket holds a key left building by another node for // a module the indexer no longer has; the program, at start, fails it naming why (issue 1660, ADR 0478 clause 6). func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) { b := startBed(t) - data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "n2.mesh-delivery-builder", Manifest: strings.Repeat("a", 64)}) + data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "n2.mesh-delivery-builder", Manifest: strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } @@ -814,7 +821,7 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) { t.Fatal(err) } b.start(t, "n1", nil) - v := b.await(t, "ghost", isState(delivery.BuildFailed)) + v := b.await(t, "ghost", isState(builds.BuildFailed)) if !strings.Contains(v.Error, "no longer has this fingerprint") || !strings.Contains(v.Error, "n2.mesh-delivery-builder") { t.Errorf("build = %+v, want it closed naming who left it", v) } @@ -829,8 +836,8 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes b := startBed(t) other := "src2:" + strings.Repeat("e", 64) digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: commit, FP: "src1:" + strings.Repeat("f", 64), FP2: other, - Manifest: digest, Path: "modules/netcheck", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}}) + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: other, + Manifest: digest, Path: "modules/netcheck", Toolchains: build.CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})}) if err != nil { t.Fatal(err) } @@ -838,14 +845,14 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes t.Fatal(err) } const head = "1111111111111111111111111111111111111111" - pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"}, - Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}} - hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) - cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"netcheck"}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"netcheck": {Fp2: fp, Manifest: digest}}}} + hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) + cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } - ckey, err := delivery.CandidatesKey("feat/x") + ckey, err := keys.EscapeName("feat/x"), error(nil) if err != nil { t.Fatal(err) } @@ -853,7 +860,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes t.Fatal(err) } b.start(t, "n1", nil) - v := b.await(t, "netcheck", isState(delivery.BuildDone)) + v := b.await(t, "netcheck", isState(builds.BuildDone)) if len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) } @@ -881,7 +888,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 digest := b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit) // The trunk carries `other`; its key is built and published. trunk := func(fp2 string) { - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: commit, FP: "src1:" + strings.Repeat("f", 64), FP2: fp2, + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2, Manifest: digest, Path: "modules/imaged"}) if err != nil { t.Fatal(err) @@ -892,12 +899,12 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 } trunk(other) b.start(t, "n1", nil) - okey, _ := delivery.BuilderBuildKey("imaged", other) - awaitKey := func(key string, pred func(delivery.BuilderBuild) bool) delivery.BuilderBuild { - var last delivery.BuilderBuild + okey, _ := builds.BuilderBuildKey("imaged", other) + awaitKey := func(key string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { + var last builds.BuilderBuild for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if e, err := b.builds.Get(b.ctx, key); err == nil { - if last, err = delivery.DecodeBuilderBuild(e.Value()); err == nil && pred(last) { + if last, err = builds.DecodeBuilderBuild(e.Value()); err == nil && pred(last) { return last } } @@ -905,24 +912,24 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 t.Fatalf("builds.%s = %+v within 20 s", key, last) return last } - v := awaitKey(okey, isState(delivery.BuildDone)) + v := awaitKey(okey, isState(builds.BuildDone)) if len(v.Artifacts) != 1 || v.Artifacts[0].Digest != digestOf("server") || !strings.HasSuffix(v.Artifacts[0].Reference, "@"+digestOf("server")) { t.Fatalf("trunk build = %+v, want the digest and the reference", v) } // The candidate carries fp: built, not published. const head = "1111111111111111111111111111111111111111" - pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"}, - Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}} - hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) - cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"imaged"}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"imaged": {Fp2: fp, Manifest: digest}}}} + hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) + cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } - ckey, _ := delivery.CandidatesKey("feat/x") + ckey := keys.EscapeName("feat/x") if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil { t.Fatal(err) } - v = b.await(t, "imaged", isState(delivery.BuildDone)) + v = b.await(t, "imaged", isState(builds.BuildDone)) if len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { t.Fatalf("candidate build = %+v, want done with no reference", v) } @@ -933,8 +940,8 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 b.fake.mu.Unlock() // The pull request is merged: the trunk's record names fp, and its key ends published, built once more with publish. trunk(fp) - v = b.await(t, "imaged", func(v delivery.BuilderBuild) bool { - return v.State == delivery.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Reference != "" + v = b.await(t, "imaged", func(v builds.BuilderBuild) bool { + return v.State == builds.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Reference != "" }) if v.Artifacts[0].Digest != digestOf("server") { t.Errorf("merged build = %+v", v) @@ -960,13 +967,13 @@ func TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain(t *testing.T) b.fake.refusePush.Store(true) b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) - v := b.await(t, "ca-trust", isState(delivery.BuildFailed)) - if !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") { - t.Fatalf("build = %+v, want failed opening %q with the store's words", v, delivery.NotStored) + v := b.await(t, "ca-trust", isState(builds.BuildFailed)) + if !strings.HasPrefix(v.Error, builds.NotStored) || !strings.Contains(v.Error, "507") { + t.Fatalf("build = %+v, want failed opening %q with the store's words", v, builds.NotStored) } b.fake.refusePush.Store(false) b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef") - v = b.await(t, "ca-trust", isState(delivery.BuildDone)) + v = b.await(t, "ca-trust", isState(builds.BuildDone)) if ok, missing := v.Referenced(); !ok { t.Errorf("build = %+v, artifacts naming no reference: %v", v, missing) } @@ -979,7 +986,7 @@ func TestAPinnedUpstreamIsCopiedByTheImageBuildModule(t *testing.T) { b := startBed(t) b.moduleCommit(t, "hello-web", "catalogue-49caf17f-hello-web.module.json", "modules/hello-web", nil, commit) b.start(t, "n1", nil) - v := b.await(t, "hello-web", isState(delivery.BuildDone)) + v := b.await(t, "hello-web", isState(builds.BuildDone)) if ok, missing := v.Referenced(); !ok { t.Fatalf("build = %+v, artifacts naming no reference: %v", v, missing) } @@ -998,10 +1005,10 @@ func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) { b := startBed(t) b.module(t, "claude-code", "catalogue-835bf22b-claude-code.module.json", "modules/claude-code") b.start(t, "n1", nil) - v := b.await(t, "claude-code", func(v delivery.BuilderBuild) bool { - return v.State == delivery.BuildDone || v.State == delivery.BuildFailed + v := b.await(t, "claude-code", func(v builds.BuilderBuild) bool { + return v.State == builds.BuildDone || v.State == builds.BuildFailed }) - if v.State != delivery.BuildDone { + if v.State != builds.BuildDone { t.Errorf("build = %+v, want done: the manifest is read", v) } } @@ -1053,7 +1060,7 @@ func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) { defer s.Unsubscribe() asked := b.asksPut(t, sub) b.start(t, "n1", nil) - v := b.await(t, "netcheck", isState(delivery.BuildDone)) + v := b.await(t, "netcheck", isState(builds.BuildDone)) for _, a := range v.Artifacts { if a.Reference == "" { t.Errorf("artifact %s names no reference", a.Name) @@ -1069,7 +1076,7 @@ func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) { t.Fatalf("asks put = %+v, want one Go and one Node delivery ask", got) } for _, a := range got { - if a.Kind != delivery.AskDelivery || a.Build == nil || a.Build.FP != fp { + if a.Kind != sdkstep.AskDelivery || a.Build == nil || a.Build.Fp != fp { t.Errorf("ask %+v, want a delivery ask of the fingerprint", a) } } @@ -1077,12 +1084,12 @@ func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) { // asksPut hears every ask put on a build seat's accept subjects (a core subscription beside the stream) and answers the // asks heard so far. -func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []delivery.StepAsk { +func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []sdkstep.StepAsk { t.Helper() var mu sync.Mutex - var asks []delivery.StepAsk + var asks []sdkstep.StepAsk s, err := nc.Subscribe("mesh.seat.*.accept.>", func(m *nats.Msg) { - if a, err := delivery.DecodeStepAsk(m.Data); err == nil { + if a, err := sdkstep.DecodeStepAsk(m.Data); err == nil { mu.Lock() asks = append(asks, a) mu.Unlock() @@ -1092,10 +1099,10 @@ func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []delivery.StepAsk { t.Fatal(err) } t.Cleanup(func() { _ = s.Unsubscribe() }) - return func() []delivery.StepAsk { + return func() []sdkstep.StepAsk { mu.Lock() defer mu.Unlock() - return append([]delivery.StepAsk(nil), asks...) + return append([]sdkstep.StepAsk(nil), asks...) } } @@ -1111,7 +1118,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { defer sub.Close() asked := b.asksPut(t, sub) b.start(t, "n1", nil) - v := b.await(t, "netcheck", func(v delivery.BuilderBuild) bool { return v.State == delivery.BuildBuilding && len(v.Asks) == 2 }) + v := b.await(t, "netcheck", func(v builds.BuilderBuild) bool { return v.State == builds.BuildBuilding && len(v.Asks) == 2 }) for deadline := time.Now().Add(20 * time.Second); len(asked()) < 2; time.Sleep(20 * time.Millisecond) { if time.Now().After(deadline) { t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked()) @@ -1120,15 +1127,15 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { // 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can // still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream. held := heldOnTheBus{b.js} - goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery} - stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery} - subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery) + goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} + stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} + subject, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery) if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil { t.Fatal(err) } queued := func() []build.QueuedState { - q, err := held.Queue(b.ctx, delivery.AskGo, 10) - if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery { + q, err := held.Queue(b.ctx, sdkstep.AskGo, 10) + if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != sdkstep.AskDelivery { t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err) } return q.Kinds[0].Messages @@ -1148,7 +1155,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { b.stop() b.start(t, "n1", nil) // the same builder, started again, while both asks wait b.fake.open() - done := b.await(t, "netcheck", isState(delivery.BuildDone)) + done := b.await(t, "netcheck", isState(builds.BuildDone)) if got := asked(); len(got) != 2 { t.Fatalf("asks put = %d (%+v), want the two the key named and none again", len(got), got) } @@ -1174,7 +1181,7 @@ type registry struct { b *bed versions, findings, reasks jetstream.KeyValue mu sync.Mutex - numbered []delivery.VersionNumbered + numbered []builds.VersionNumbered removes []map[string]any } @@ -1193,12 +1200,12 @@ func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *regis // EVENTS with two seats' outcomes, and the forge bridge's stream (a branch's end): each with the builder's durable consumer. for stream, subjects := range map[string][]string{ eventsStream: {"mesh.mod.*.event.>", "mesh.seat.*.event.>"}, - forgeStream: {process.ModuleStreamSubject(delivery.ForgeBridgeModule, ">")}, + forgeStream: {process.ModuleStreamSubject(forgeBridgeModule, ">")}, } { if _, err := b.js.CreateOrUpdateStream(b.ctx, jetstream.StreamConfig{Name: stream, Subjects: subjects}); err != nil { t.Fatal(err) } - if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream, jetstream.ConsumerConfig{Durable: node + "_" + delivery.BuilderModule, + if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream, jetstream.ConsumerConfig{Durable: node + "_" + builds.BuilderModule, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } @@ -1218,8 +1225,8 @@ func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *regis t.Cleanup(func() { _ = rm.Unsubscribe() }) nc := b.js.Conn() // As a build module hears it: on the contract's subject, read by the contract's decoder (BG2's late publish). - sub, err := nc.Subscribe(delivery.VersionNumberedSubject, func(m *nats.Msg) { - if nb, err := delivery.DecodeVersionNumbered(m.Data); err == nil { + sub, err := nc.Subscribe(builds.VersionNumberedSubject, func(m *nats.Msg) { + if nb, err := builds.DecodeVersionNumbered(m.Data); err == nil { r.mu.Lock() r.numbered = append(r.numbered, nb) r.mu.Unlock() @@ -1234,9 +1241,9 @@ func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *regis listed = append(listed, map[string]string{"version": v}) } for subject, answer := range map[string]any{ - "mesh.seat." + ociartifactregistry.Seat + ".call.versions.>": map[string]any{"versions": listed}, - "mesh.mod.mesh-delivery-forge-bridge.call.bridge_tags.>": map[string]any{"tags": []string{}}, - "mesh.mod." + delivery.IndexerModule + ".call.indexer_requirements.>": map[string]any{"requirements": []string{}}, + "mesh.seat." + ociartifactregistry.Seat + ".call.versions.>": map[string]any{"versions": listed}, + "mesh.mod.mesh-delivery-forge-bridge.call.bridge_tags.>": map[string]any{"tags": []string{}}, + "mesh.mod." + indexerModule + ".call.indexer_requirements.>": map[string]any{"requirements": []string{}}, } { reply, _ := json.Marshal(map[string]any{"result": answer, "node": "ctl"}) s, err := nc.Subscribe(subject, func(m *nats.Msg) { _ = m.Respond(reply) }) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 5559b78b..3e614c9d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -8,7 +8,6 @@ import ( "cmp" "context" "fmt" - sdkfindings "git.novox.be/novox/mesh-delivery/sdk/findings" "io" "log" "maps" @@ -20,14 +19,20 @@ import ( "syscall" "time" + sdkfindings "git.novox.be/novox/mesh-delivery/sdk/findings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/forms" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" @@ -63,7 +68,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer conn.Close() - builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key") + buildsKV, err := process.EssentialBucket(ctx, js, "builder", builds.BuilderBuildsBucket, "claim or write a build key") if err != nil { return err } @@ -79,28 +84,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The module has one durable consumer of EVENTS: one hub pulls it and hands each message to the part that hears it (the // views reader and the builder's own events), started once every part is registered. hub := process.NewEventHub(logger.Printf) - reader := &views.ProcessReader{Module: delivery.BuilderModule, Call: calls.Call, Logf: logger.Printf, Hub: hub} + reader := &views.ProcessReader{Module: builds.BuilderModule, Call: calls.Call, Logf: logger.Printf, Hub: hub} findingsKV := process.AwaitBucket(ctx, js, conn, findingsBucket, "the builder's findings, a first view unanswered past its bound among them, are said in its log only", logger.Printf) reader.Findings = func() views.FindingWriter { return sdkfindings.Writer(ctx, findingsKV.KV) } - modules, err := reader.Follow(delivery.IndexerModule, modulesState, delivery.IndexerModule+".indexer_read_modules") + modules, err := reader.Follow(indexerModule, modulesState, indexerModule+".indexer_read_modules") if err != nil { return err } - manifests, err := reader.Follow(delivery.IndexerModule, manifestsState, delivery.IndexerModule+".indexer_read_manifests") + manifests, err := reader.Follow(indexerModule, manifestsState, indexerModule+".indexer_read_manifests") if err != nil { return err } // The candidates' fingerprints are built too (issue 1684). - candidates, err := reader.Follow(delivery.IndexerModule, candidatesState, delivery.IndexerModule+".indexer_read_candidates") + candidates, err := reader.Follow(indexerModule, candidatesState, indexerModule+".indexer_read_candidates") if err != nil { return err } // The indexer's packages, whose names a package is numbered under (design 32). - packages, err := reader.Follow(delivery.IndexerModule, delivery.PackagesState, delivery.IndexerModule+".indexer_read_packages") + packages, err := reader.Follow(indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages") if err != nil { return err } - placement, err := reader.Follow(delivery.AssignerModule, delivery.PlacementState, delivery.AssignerModule+".assigner_read_placement") + placement, err := reader.Follow(assignerModule, "placement", assignerModule+".assigner_read_placement") if err != nil { return err } @@ -128,15 +133,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dead := &buildrun.DeadLetters{} // The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it // is bound when the bus says it was created, and the latest hearing is written then. - hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf) - hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf, + hearingKV := process.AwaitBucket(ctx, js, conn, builds.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf) + hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, buildstep.AskLanguages, dead, logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf)) defer hearing.Stop() // Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read // (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28). var buildModules []string - for _, l := range delivery.AskLanguages { - if m, err := delivery.BuildModuleOf(l); err == nil { + for _, l := range buildstep.AskLanguages { + if m, err := buildstep.BuildModuleOf(l); err == nil { buildModules = append(buildModules, m) } } @@ -158,7 +163,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32). numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, Names: versions.NamesIn(indexerPackages{packages}), - Said: func(nb delivery.VersionNumbered) { + Said: func(nb builds.VersionNumbered) { if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil { logger.Printf("%v", err) } @@ -169,7 +174,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf} numberer.Publishing = pubs.arm b := &build.Builder{ - Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, + Claimer: build.Claimer{Store: kvStore{buildsKV}, By: cred.Node + "." + cred.Module}, Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound}, Held: heldOnTheBus{js}, // An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound. @@ -189,7 +194,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // An ask not answered within five minutes is narrated with what it waits on: its bound's end on the bus says when. boundsKV := process.AwaitBucket(ctx, js, conn, versions.TagAskBoundsBucket, "an unanswered tag ask is not narrated", logger.Printf) bounds := &tagBounds{asks: tagAsksKV.KV, bounds: boundsKV.KV, bound: tagAskBound, waits: floorWaits.Waiting, logf: logger.Printf, now: time.Now, - log: func(e delivery.BuildLogEntry) { b.Log(e) }} + log: func(e builds.BuildLogEntry) { b.Log(e) }} // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. @@ -218,7 +223,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.Trunk = trunks.head // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the // indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since. - b.Log = func(e delivery.BuildLogEntry) { + b.Log = func(e builds.BuildLogEntry) { if e.Node == "" { // a narration of no one build (a tag ask, the old form's window) is this instance's e.Node = cred.Node } @@ -262,7 +267,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The old reference form's window over them (DLV-B9), kept in the state old-form once it is bound; builder_old_form // reads it and drops a retired node by hand. reads.ctx = context.WithoutCancel(ctx) - reads.window = &oldForm{forms: forms.Current, findings: found, logf: logger.Printf, now: time.Now, log: func(e delivery.BuildLogEntry) { b.Log(e) }, + reads.window = &oldForm{forms: forms.Current, findings: found, logf: logger.Printf, now: time.Now, log: func(e builds.BuildLogEntry) { b.Log(e) }, bounds: pubBoundsKV.KV, said: func(body versions.OldFormClosedBody) { if err := emitEvent(context.WithoutCancel(ctx), js, cred, versions.OldFormClosedEvent, "old-form-closed-"+body.At.Format(time.RFC3339Nano), body); err != nil { @@ -432,7 +437,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }{ {forgeStream, "a branch's end", func(m jetstream.Msg) { ends.heard(ctx, m) }}, {checkerAsksStream, "the checker's tag-wanted", func(m jetstream.Msg) { - tagWanted(ctx, tagAsksKV.KV(), m, delivery.CheckerModule, time.Now(), logger.Printf) + tagWanted(ctx, tagAsksKV.KV(), m, checkerModule, time.Now(), logger.Printf) }}, } { hear(c.stream, c.what+" is not heard until the builder starts again", c.hear) @@ -461,9 +466,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder's own state of its first views (views.StatusState); its findings (findingsBucket) are where the condition // keeper raises a first view unanswered past its bound (ADR 0615 §6). -var firstViewsBucket = delivery.BuilderModule + "_" + views.StatusState +var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState -// The indexer's states the builder follows (delivery.ModulesBucket, ManifestsBucket, CandidatesBucket), and the owners' +// The indexer's states the builder follows (indexerBucket(modulesState), ManifestsBucket, CandidatesBucket), and the owners' // views families as their manifests' `streams` name them. const ( modulesState = "modules" @@ -513,7 +518,7 @@ func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, } // sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest -// delivery.SourceHistory of them: what a candidate's base revision names. +// sourceHistory of them: what a candidate's base revision names. type sourceHeads struct { mu sync.Mutex byRev map[uint64]map[string]string @@ -524,7 +529,7 @@ func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool { if !opOf(e) { return false } - src, err := delivery.DecodeSource(e.Value()) + src, err := meshdeliveryclient.DecodeSource(e.Value()) if err != nil { return false } @@ -537,7 +542,7 @@ func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool { return false } s.byRev[e.Revision()] = src.Heads - for len(s.byRev) > delivery.SourceHistory { + for len(s.byRev) > sourceHistory { delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev)))) } return true @@ -552,7 +557,7 @@ func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (st return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev) } for r, head := range heads { - if delivery.SameRepository(r, repository) { + if trunk.SameRepository(r, repository) { return head, nil } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go new file mode 100644 index 00000000..5c425e7b --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go @@ -0,0 +1,38 @@ +package main + +import ( + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" +) + +// The holders and buckets the builder names, taken from its own manifest (module.json: its `reads`) and from the holders' +// seats' state names; a unit row checks them against the manifest (TestTheNamesAreTheManifests). No contract names a holder +// (novox/hq ADR 0625 §4, ADR 0634 §1), except the clients of the indexer's and the assigner's states, whose holder names +// are used from there. +const ( + gateModule = "mesh-delivery-gate" + checkerModule = "mesh-delivery-checker" + forgeBridgeModule = "mesh-delivery-forge-bridge" + + // indexerModule is the holder of the states modules, manifests, candidates and packages the builder follows. + indexerModule = meshcatalogclient.Module + // assignerModule is the holder of the state placement. + assignerModule = buildrun.AssignerModule + + // sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names. + sourceBucket = gateModule + "_source" + // packagesBucket is the indexer's state packages. + packagesBucket = indexerModule + "_" + meshcatalog.StatePackages + + // The revisions each keeps per key (the beds of the integration test create them alike). + sourceHistory = 64 + candidatesHistory = 10 + modulesHistory = 20 + manifestsHistory = 1 +) + +func indexerBucket(state string) string { return indexerModule + "_" + state } + +// namedTreePrefix opens the tree input of a module whose every artifact names its paths. +const namedTreePrefix = "named:" diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go new file mode 100644 index 00000000..544f3090 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "encoding/json" + "os" + "slices" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" +) + +// TestTheNamesAreTheManifests: the holders and buckets the builder names are its manifest's `reads` (the gate's source, the +// build modules' steps and toolchains) and its `consumes` of the indexer's and assigner's views, since no contract names a +// holder (novox/hq ADR 0634 §1). +func TestTheNamesAreTheManifests(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Reads []string `json:"reads"` + Consumes []string `json:"consumes"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, read := range []string{ + gateModule + ".source", + "mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", + } { + if !slices.Contains(m.Reads, read) { + t.Errorf("the builder reads %s, its manifest's reads are %v", read, m.Reads) + } + } + for _, view := range []string{indexerModule + ".views.modules", indexerModule + ".views.manifests", indexerModule + ".views.candidates", + indexerModule + ".views.packages", assignerModule + ".views.placement"} { + if !slices.Contains(m.Consumes, view) { + t.Errorf("the builder follows %s, its manifest consumes %v", view, m.Consumes) + } + } + for want, got := range map[string]string{ + "mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), + "mesh-build-node_toolchains": buildrun.ToolchainsBucket("mesh-build-node"), "mesh-delivery-indexer_packages": packagesBucket, + "mesh-delivery-assigner_placement": buildrun.PlacementBucket, + } { + if want != got { + t.Errorf("a bucket is %q, want %q", got, want) + } + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/oldform.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/oldform.go index 20dfb4e5..fa1189ce 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/oldform.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/oldform.go @@ -12,7 +12,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-delivery/sdk/forms" @@ -30,7 +30,7 @@ type oldForm struct { // said emits version.old-form-closed. said func(versions.OldFormClosedBody) // log writes the close into the builder's build log (build.said); nil writes none. - log func(delivery.BuildLogEntry) + log func(builds.BuildLogEntry) logf func(string, ...any) now func() time.Time @@ -85,7 +85,7 @@ func (o *oldForm) dropped() map[string]bool { // step moves the window to what counted states. // named is every node the read named (the placement's nodes before any is dropped), counted the statements of those -// still counted: a read naming no node judges nothing (delivery.StepWindow), so it never closes the window. +// still counted: a read naming no node judges nothing (window.StepWindow), so it never closes the window. func (o *oldForm) step(ctx context.Context, named []string, counted map[string][]string) { o.mu.Lock() defer o.mu.Unlock() @@ -119,7 +119,7 @@ func (o *oldForm) step(ctx context.Context, named []string, counted map[string][ // In the build log too (design 51 *The node-engine's own build*), under the builder's own key, the window's key // standing for the fingerprint: the close is no one build's. if o.log != nil { - o.log(delivery.BuildLogEntry{Event: delivery.BuildSaidEvent, Module: delivery.BuilderModule, FP: versions.OldFormKey, + o.log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: builds.BuilderModule, FP: versions.OldFormKey, At: body.At.UTC(), Line: line, Why: versions.SaidOldFormClosed}) } if o.said != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go index 02afe10c..09324c26 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go @@ -7,7 +7,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" ) // **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of @@ -39,17 +39,17 @@ func (t *trunkSource) ensure() error { if t.opened { return nil } - kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket) + kv, err := t.js.KeyValue(t.ctx, sourceBucket) if err != nil { - return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err) + return fmt.Errorf("the bucket %s cannot be read (%v)", sourceBucket, err) } - w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory()) + w, err := kv.Watch(t.ctx, meshdeliveryclient.SourceMainKey, jetstream.IncludeHistory()) if err != nil { - return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err) + return fmt.Errorf("the bucket %s cannot be watched (%v)", sourceBucket, err) } sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) }) if err != nil { - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) + return fmt.Errorf("the bucket %s: %w", sourceBucket, err) } t.opened = true go func() { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go index a4697771..c2c12614 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go @@ -7,7 +7,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -22,7 +22,7 @@ import ( const EngineReadsState = "engine-reads" // engineReadsBucket is that state's bucket. -const engineReadsBucket = delivery.BuilderModule + "_" + EngineReadsState +const engineReadsBucket = builds.BuilderModule + "_" + EngineReadsState // engineReads keeps the statements and the counted nodes for the builder's forms (build.Builder.Forms). type engineReads struct { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 3e57a46a..8d7c8d3c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -8,8 +8,8 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/buscompose" - "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -134,18 +134,18 @@ func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e } // appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key -// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential. +// (builds.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential. // The id makes a repeated append of one entry one entry. -func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error { - data, err := delivery.EncodeBuildLogEntry(e) +func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e builds.BuildLogEntry) error { + data, err := builds.EncodeBuildLogEntry(e) if err != nil { return err } - key, err := delivery.BuildLogKey(e.Module) + key, err := builds.BuildLogKey(e.Module) if err != nil { return err } - msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key) + msg := nats.NewMsg(buscompose.LogSubject(cred.Module, builds.BuilderLog) + "." + key) msg.Data = data id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano) pub, cancel := context.WithTimeout(ctx, 10*time.Second) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index cc000338..2e36a1db 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -21,11 +21,14 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" + "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/sdk/enginereads" "git.novox.be/novox/mesh-delivery/sdk/forms" - "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -57,7 +60,7 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { // head is told why and opens it again then. src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} src.start() - if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { + if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), sourceBucket) { t.Errorf("a trunk head with the source refused: %v, want why", err) } if src.opened { @@ -74,11 +77,11 @@ func theCountedNodesEngineReads(t *testing.T) { gate := buildrun.NewToolchainGate(nil, func(string, string, []byte, bool) bool { return false }, t.Logf) place := func(nodes ...string) { for _, node := range []string{"n1", "n2"} { - pl := delivery.Placement{Node: node} + pl := meshmoduleassigner.Placement{Node: node} if slices.Contains(nodes, node) { - pl.Modules = []delivery.PlacedModule{{Module: "web"}} + pl.Modules = []meshmoduleassigner.PlacedModule{{Module: "web"}} } - values, err := delivery.PlacementValues(pl) + values, err := meshmoduleassignerclient.PlacementValues(pl) if err != nil { t.Fatal(err) } @@ -86,7 +89,7 @@ func theCountedNodesEngineReads(t *testing.T) { gate.Placement(key, raw, false, time.Now()) } if !slices.Contains(nodes, node) { - key, _ := delivery.PlacementModuleKey(node, "web") + key, _ := meshmoduleassignerclient.ModuleKey(node, "web") gate.Placement(key, nil, true, time.Now()) } } @@ -264,11 +267,11 @@ func aBranchsEndIsHeard(t *testing.T) { f forgeEnd want bool }{ - {forgeEnd{Kind: delivery.ForgeMerged, Repository: "o/r", Branch: "feat/x"}, true}, + {forgeEnd{Kind: forgemsg.ForgeMerged, Repository: "o/r", Branch: "feat/x"}, true}, {forgeEnd{Kind: forgeBranchDeleted, Repository: "o/r", Branch: "feat/x"}, true}, - {forgeEnd{Kind: delivery.ForgePull, Repository: "o/r", Branch: "feat/x", Action: delivery.PullClosed}, true}, - {forgeEnd{Kind: delivery.ForgePull, Repository: "o/r", Branch: "feat/x", Action: delivery.PullOpened}, false}, - {forgeEnd{Kind: delivery.ForgeMerged, Repository: "o/r"}, false}, + {forgeEnd{Kind: forgemsg.ForgePull, Repository: "o/r", Branch: "feat/x", Action: forgemsg.PullClosed}, true}, + {forgeEnd{Kind: forgemsg.ForgePull, Repository: "o/r", Branch: "feat/x", Action: forgemsg.PullOpened}, false}, + {forgeEnd{Kind: forgemsg.ForgeMerged, Repository: "o/r"}, false}, } { if got := c.f.ended(); got != c.want { t.Errorf("%+v ended = %v, want %v", c.f, got, c.want) @@ -290,7 +293,7 @@ func aBranchsEndIsHeard(t *testing.T) { verdict := &wakeMsg{subject: "mesh.stream.mesh-delivery-checker.event.verdict", data: []byte(`{"repository":"o/r"}`)} wanted := &wakeMsg{subject: tagWantedSubject, data: []byte(`{"repository":"o/r","branch":"feat/x","head":"abc","tag":"feat-x"}`)} for _, m := range []*wakeMsg{verdict, wanted} { - tagWanted(context.Background(), nil, m, delivery.CheckerModule, time.Now(), t.Logf) + tagWanted(context.Background(), nil, m, checkerModule, time.Now(), t.Logf) } if !verdict.acked || verdict.delayed != 0 || wanted.acked || wanted.delayed != versions.RemoveAgainAfter || !strings.HasPrefix(tagWantedSubject, "mesh.stream.") { t.Errorf("a verdict acked %v (delayed %v), a tag-wanted acked %v (delayed %v)", verdict.acked, verdict.delayed, wanted.acked, wanted.delayed) @@ -313,20 +316,20 @@ func (failingVersions) KeysOf(context.Context, string) ([]string, error) { func aFailedTrunkBuildIsAFinding(t *testing.T) { st := &memFindings{vals: map[string][]byte{}} f := &findings{store: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} - ended := func(outcome, ref string) delivery.BuildLogEntry { - return delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Ref: ref, Outcome: outcome} + ended := func(outcome, ref string) builds.BuildLogEntry { + return builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Ref: ref, Outcome: outcome} } key := health.FindingKey(health.ScopeModule, "m", BuildFailed) - f.buildEnded(context.Background(), ended(delivery.BuildFailed, "refs/pull/3/head")) + f.buildEnded(context.Background(), ended(builds.BuildFailed, "refs/pull/3/head")) if len(st.vals) != 0 { t.Fatalf("a candidate's failed build is a finding: %v", st.vals) } - f.buildEnded(context.Background(), ended(delivery.BuildFailed, "")) + f.buildEnded(context.Background(), ended(builds.BuildFailed, "")) got, err := health.DecodeFinding(st.vals[key]) - if err != nil || got.Observer != delivery.BuilderModule || !strings.Contains(got.Summary, "trunk build of m at f1") { + if err != nil || got.Observer != builds.BuilderModule || !strings.Contains(got.Summary, "trunk build of m at f1") { t.Fatalf("the finding = %+v, %v", got, err) } - f.buildEnded(context.Background(), ended(delivery.BuildDone, "")) + f.buildEnded(context.Background(), ended(builds.BuildDone, "")) if _, ok := st.vals[key]; ok { t.Error("a later done trunk build leaves build-failed") } @@ -375,10 +378,10 @@ func aFailedTrunkBuildIsAFinding(t *testing.T) { t.Fatalf("a bound not written: the finding = %+v, %v", got, err) } watchBounds(context.Background(), refusedWatch{err: errors.New("permissions violation")}, func(jetstream.KeyValueEntry) {}, f.notHeard(context.Background()), nil, t.Logf) - if got, err := health.DecodeFinding(st.vals[heard.Key(delivery.BuilderModule, versions.BoundsState)]); err != nil || !strings.Contains(got.Summary, "permissions violation") { + if got, err := health.DecodeFinding(st.vals[heard.Key(builds.BuilderModule, versions.BoundsState)]); err != nil || !strings.Contains(got.Summary, "permissions violation") { t.Fatalf("bounds not watched: the finding = %+v, %v", got, err) } - delete(st.vals, heard.Key(delivery.BuilderModule, versions.BoundsState)) + delete(st.vals, heard.Key(builds.BuilderModule, versions.BoundsState)) malformed := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"version":"0.1.4"}`)} unreadKey := health.FindingKey(health.ScopeModule, "oci-artifact-registry", versions.SeatOutcomeUnread) if !h.heard(context.Background(), malformed) || !malformed.acked || st.vals[unreadKey] == nil { @@ -390,7 +393,7 @@ func aFailedTrunkBuildIsAFinding(t *testing.T) { } // ADR 0615 §5 (review round 1): a consumer not there yet is family-not-heard naming its stream, cleared when it binds. notHeard := f.notHeard(context.Background()) - unheardKey := heard.Key(delivery.BuilderModule, "FORGE") + unheardKey := heard.Key(builds.BuilderModule, "FORGE") _ = notHeard.Bind("FORGE", "a branch's end is not heard", func() error { return errors.New("consumer not found") }) if got, err := health.DecodeFinding(st.vals[unheardKey]); err != nil || got.Kind != heard.Kind || !strings.Contains(got.Summary, "does not hear FORGE") { t.Fatalf("a consumer not there: the finding = %+v, %v", got, err) @@ -405,11 +408,11 @@ func aFailedTrunkBuildIsAFinding(t *testing.T) { now := time.Unix(0, 0) fs := forms.ArtifactForms{{Name: "registry", First: true, Kinds: []string{"image"}}, {Name: "oci", Kinds: []string{"image"}}} var closings []versions.OldFormClosedBody - var logged []delivery.BuildLogEntry + var logged []builds.BuildLogEntry marks := &memBounds{keys: map[string]bool{}} window := &oldForm{forms: fs, findings: f, logf: t.Logf, now: func() time.Time { return now }, bounds: func() jetstream.KeyValue { return marks }, said: func(b versions.OldFormClosedBody) { closings = append(closings, b) }, - log: func(e delivery.BuildLogEntry) { logged = append(logged, e) }} + log: func(e builds.BuildLogEntry) { logged = append(logged, e) }} reads := &engineReads{keeper: &enginereads.Keeper{}, window: window, counted: []string{"a", "b"}} newest := []string{forms.FormNeed("oci")} counted := map[string][]string{"a": newest, "b": nil} @@ -442,10 +445,10 @@ func aFailedTrunkBuildIsAFinding(t *testing.T) { if window.w.Open || len(closings) != 1 || !slices.Equal(closings[0].LastHolding, []string{"b"}) || st.vals[heldKey] != nil { t.Errorf("the node back with the newest form: %+v, closings %+v, finding left %v", window.w, closings, st.vals[heldKey] != nil) } - if len(logged) != 1 || logged[0].Event != delivery.BuildSaidEvent || logged[0].Why != versions.SaidOldFormClosed || + if len(logged) != 1 || logged[0].Event != builds.BuildSaidEvent || logged[0].Why != versions.SaidOldFormClosed || !strings.Contains(logged[0].Line, "last held by b") { t.Errorf("the build log = %+v, want the close said once (build.said)", logged) - } else if _, err := delivery.EncodeBuildLogEntry(logged[0]); err != nil { + } else if _, err := builds.EncodeBuildLogEntry(logged[0]); err != nil { t.Errorf("the close is no entry the build log takes: %v", err) } if _, err := window.drop(context.Background(), "b", "op@shanks", "retired", now); err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go index 694f6ba8..d6029f81 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go @@ -8,10 +8,10 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" - "git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry" - "git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry" - "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go index 4c6d3618..c9dc9d67 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go @@ -8,7 +8,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "mesh-delivery-builder/internal/versions" ) @@ -22,7 +22,7 @@ type tagBounds struct { waits func() []versions.FloorWait logf func(string, ...any) // log writes a narration into the builder's build log (build.said); nil writes none. - log func(delivery.BuildLogEntry) + log func(builds.BuildLogEntry) now func() time.Time bound time.Duration } @@ -82,7 +82,7 @@ func (tb *tagBounds) ended(ctx context.Context, e jetstream.KeyValueEntry) { // In the build log too (design 51 *Tagged versions*): under the builder's own key, no build being claimed yet, the // ask's key standing for the fingerprint. if tb.log != nil { - tb.log(delivery.BuildLogEntry{Event: delivery.BuildSaidEvent, Module: delivery.BuilderModule, FP: e.Key(), Commit: t.Head, + tb.log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: builds.BuilderModule, FP: e.Key(), Commit: t.Head, At: tb.now().UTC(), Line: line, Why: versions.SaidUnanswered}) } tb.arm(ctx, e.Key()) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 38a9259c..82734a16 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -12,8 +12,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/delivery" ) // toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it @@ -57,7 +55,7 @@ func (w *toolchainWatch) ensure(module string) error { if w.watched[module] { return nil } - name := delivery.ToolchainsBucket(module) + name := buildrun.ToolchainsBucket(module) kv, err := w.js.KeyValue(w.ctx, name) switch { case errors.Is(err, jetstream.ErrBucketNotFound): diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index f2bd2cd3..de3e51e1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -11,9 +11,9 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" @@ -98,7 +98,7 @@ func followPackages(ctx context.Context, kv jetstream.KeyValue, b *build.Builder module, name, named := versions.NamedModule(e.Key(), e.Value()) if named && heard[e.Key()] != module+" "+name { heard[e.Key()] = module + " " + name - logf("%s.%s names a package of %s: its build is looked at again", delivery.PackagesBucket, e.Key(), module) + logf("%s.%s names a package of %s: its build is looked at again", packagesBucket, e.Key(), module) b.PackageNamed(ctx, module) } }) @@ -203,13 +203,13 @@ func floorVia(call toolCaller) versions.FloorSources { // emitNumbered says the builder's event version.numbered (design 51 *Events of the builder*), on its own family apart // from its builds' ends; the name and version are the bus's dedup id. -func emitNumbered(ctx context.Context, js jetstream.JetStream, cred process.Account, b delivery.VersionNumbered) error { +func emitNumbered(ctx context.Context, js jetstream.JetStream, cred process.Account, b builds.VersionNumbered) error { id := "numbered-" + versions.Key(b.Name, b.Version) if b.Again != "" { id += "-again-" + b.Again } - if err := emitEvent(ctx, js, cred, delivery.VersionNumberedEvent, id, b); err != nil { - return fmt.Errorf("saying %s for %s@%s: %w", delivery.VersionNumberedEvent, b.Name, b.Version, err) + if err := emitEvent(ctx, js, cred, builds.VersionNumberedEvent, id, b); err != nil { + return fmt.Errorf("saying %s for %s@%s: %w", builds.VersionNumberedEvent, b.Name, b.Version, err) } return nil } @@ -295,7 +295,7 @@ func conditionModules(id string) []string { // doneVia says whether a build key is done, read from the builder's own builds state: the build's file is held. func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (bool, error) { return func(ctx context.Context, module, fp string) (bool, error) { - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return false, err } @@ -303,11 +303,11 @@ func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (boo if err != nil || !found { return false, err } - cur, err := delivery.DecodeBuilderBuild(raw) + cur, err := builds.DecodeBuilderBuild(raw) if err != nil { return false, err } - return cur.State == delivery.BuildDone, nil + return cur.State == builds.BuildDone, nil } } diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index cfa50977..53501b9f 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,16 +3,27 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) -require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261104174638-125929dbc459 - require ( git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 @@ -31,11 +42,7 @@ require ( replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun require ( - git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639 - git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-20261104174344-ae5266e77639 git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639 - git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-20261104174344-ae5266e77639 - git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 ) diff --git a/modules/mesh-delivery-builder/internal/build/announce.go b/modules/mesh-delivery-builder/internal/build/announce.go index 7ccdd9b2..c810d215 100644 --- a/modules/mesh-delivery-builder/internal/build/announce.go +++ b/modules/mesh-delivery-builder/internal/build/announce.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a @@ -18,7 +18,7 @@ import ( // not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again). func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult { b.init() - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return BuildAgainResult{Refused: err.Error()} } @@ -29,13 +29,13 @@ func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) if !found { return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)} } - cur, err := delivery.DecodeBuilderBuild(value) + cur, err := builds.DecodeBuilderBuild(value) if err != nil { return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)} } switch cur.State { - case delivery.BuildDone: - case delivery.BuildFailed: + case builds.BuildDone: + case builds.BuildFailed: return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)} default: return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)} diff --git a/modules/mesh-delivery-builder/internal/build/announce_test.go b/modules/mesh-delivery-builder/internal/build/announce_test.go index 021c900a..64101416 100644 --- a/modules/mesh-delivery-builder/internal/build/announce_test.go +++ b/modules/mesh-delivery-builder/internal/build/announce_test.go @@ -6,19 +6,19 @@ import ( "strings" "testing" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29 for _, c := range []struct { name string - key *delivery.BuilderBuild + key *builds.BuilderBuild fp string refuse string }{ - {name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}}, - {name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}}, - {name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}}, + {name: "a done key is written again, unchanged, as a new revision", key: &builds.BuilderBuild{State: builds.BuildDone, Step: "built"}}, + {name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "x"}}, + {name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "asked"}}, {name: "a key with no build is refused", refuse: "has no build key"}, {name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"}, } { diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go index e91e6ba2..aa9147d3 100644 --- a/modules/mesh-delivery-builder/internal/build/askagain.go +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -12,12 +12,13 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buildstep" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up -// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or +// once it was delivered as often as its worker may (sdkstep.AskMaxDeliver), and a message given up on, dropped or // terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on // restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound), // a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new @@ -90,10 +91,10 @@ type Taken struct { } // Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer -// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means +// delivered: a holder pulls the kinds in sdkstep.AskKinds' order, so taking one of the same kind or of a later one means // its worker had none of waiting's kind left to give. func Proves(taken, waiting string) bool { - t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting) + t, w := slices.Index(sdkstep.AskKinds, taken), slices.Index(sdkstep.AskKinds, waiting) return t >= 0 && w >= 0 && t >= w } @@ -162,11 +163,11 @@ func AskedAt(id string) time.Time { // Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked // again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62). type Dropper interface { - Drop(ctx context.Context, a delivery.StepAsk) error + Drop(ctx context.Context, a sdkstep.StepAsk) error } // drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put. -func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) { +func (b *Builder) drop(ctx context.Context, ask sdkstep.StepAsk, old string) { d, ok := b.Gone.(Dropper) if !ok { return @@ -302,8 +303,8 @@ func (h byHand) Error() string { return "asked again by hand: " + h.why } // answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person // asked it again (AskAgain); it answers the step and the ask that has it. -func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) { - key, _ := delivery.BuilderBuildKey(module, fp) +func (b *Builder) answer(ctx context.Context, l *lane, ask sdkstep.StepAsk, asked *askNotes, module, fp string) (sdkstep.Step, sdkstep.StepAsk, error) { + key, _ := builds.BuilderBuildKey(module, fp) again := b.listen(key, ask.Language) defer b.unlisten(key, ask.Language, again) asking := askingOf(asked.step, l.module) @@ -370,7 +371,7 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask // waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone // and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on. -func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) { +func (b *Builder) waitOnce(ctx context.Context, ask sdkstep.StepAsk, again <-chan string) (sdkstep.Step, string, error) { hand, cancel := context.WithCancelCause(ctx) defer cancel(nil) go func() { @@ -387,7 +388,7 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch defer stop() } step, err := b.Asks.Wait(wait, ask) - if _, declined := delivery.DeclinedOf(step); err == nil && declined { + if _, declined := sdkstep.DeclinedOf(step); err == nil && declined { // A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again // after a pause, within DeclineBound. return step, step.Reason, errDeclined diff --git a/modules/mesh-delivery-builder/internal/build/askfake_test.go b/modules/mesh-delivery-builder/internal/build/askfake_test.go index 6894884e..c5bcffb2 100644 --- a/modules/mesh-delivery-builder/internal/build/askfake_test.go +++ b/modules/mesh-delivery-builder/internal/build/askfake_test.go @@ -7,7 +7,7 @@ import ( "maps" "strings" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) // The build runner a build node runs an ask with is mesh-build-go's (novox/hq ADR 0593 §1), another repository's, @@ -21,18 +21,19 @@ type stepFunc func(ctx context.Context, args map[string]any) (any, error) // askSteps are a build module's steps an ask is run with. type askSteps struct{ Fetch, Build, Copy stepFunc } -// runAsk runs a build ask with steps and answers its delivery.BuildAnswer, or the error that fails the step. -func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, error) { +// runAsk runs a build ask with steps and answers its buildstep.BuildAnswer, or the error that fails the step. +func runAsk(ctx context.Context, steps askSteps, ask buildstep.StepAsk) (any, error) { if ask.Build == nil { return nil, fmt.Errorf("the ask %s holds no build job", ask.ID) } job := *ask.Build fetch := job.Fetch if len(ask.Packages) > 0 && fetch != nil { - fetch = maps.Clone(fetch) - fetch["packages"] = maps.Clone(ask.Packages) + withPackages := *fetch + withPackages.Packages = maps.Clone(ask.Packages) + fetch = &withPackages } - answer := delivery.BuildAnswer{} + answer := buildstep.BuildAnswer{} workspace := "" for _, a := range job.Artifacts { var out any @@ -44,7 +45,7 @@ func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, err out, err = steps.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From}) } else { if workspace == "" { - raw, err := asRaw(steps.Fetch(ctx, fetch)) + raw, err := asRaw(steps.Fetch(ctx, fetchArgs(fetch))) if err != nil { return nil, err } @@ -56,7 +57,7 @@ func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, err if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" { return nil, fmt.Errorf("the fetch answered no workspace: %s", raw) } - if _, candidate := fetch["base"]; candidate && (len(f.Conflicts) > 0 || f.Merge == "") { + if candidate := fetch.Base != ""; candidate && (len(f.Conflicts) > 0 || f.Merge == "") { return nil, fmt.Errorf("the head does not merge into the trunk: %s", strings.Join(f.Conflicts, ", ")) } workspace = f.Workspace @@ -78,7 +79,7 @@ func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, err if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" { return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, raw) } - answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference}) + answer.Artifacts = append(answer.Artifacts, buildstep.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference}) } return answer, nil } @@ -92,3 +93,11 @@ func asRaw(v any, err error) (json.RawMessage, error) { } return json.Marshal(v) } + +// fetchArgs is the fetch of an ask as the fetch tool is called with it: its JSON object. +func fetchArgs(f *buildstep.Fetch) map[string]any { + raw, _ := json.Marshal(f) + var args map[string]any + _ = json.Unmarshal(raw, &args) + return args +} diff --git a/modules/mesh-delivery-builder/internal/build/buildagain.go b/modules/mesh-delivery-builder/internal/build/buildagain.go index 62d3deee..6f39187e 100644 --- a/modules/mesh-delivery-builder/internal/build/buildagain.go +++ b/modules/mesh-delivery-builder/internal/build/buildagain.go @@ -8,7 +8,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again @@ -88,7 +88,7 @@ func (b *Builder) handEnded(key string) { // ctx is the builder's own context: the build it starts lives on it. func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult { b.init() - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return BuildAgainResult{Refused: err.Error()} } @@ -99,18 +99,18 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo if !found { return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)} } - cur, err := delivery.DecodeBuilderBuild(value) + cur, err := builds.DecodeBuilderBuild(value) if err != nil { return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)} } switch { - case cur.State == delivery.BuildDone: + case cur.State == builds.BuildDone: return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)} - case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued: + case cur.State == builds.BuildBuilding || cur.State == builds.BuildQueued: return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)} - case cur.State != delivery.BuildFailed: + case cur.State != builds.BuildFailed: return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)} - case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix): + case builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix): return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)} } // One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint, @@ -119,7 +119,7 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo b.mu.Lock() m, known := b.known[module] switch { - case !known || m.FP2 != fp: + case !known || m.Fp2 != fp: b.mu.Unlock() return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)} case m.Manifest == "": @@ -159,11 +159,11 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0): // The look ended before every ask was put: say how the key stands, not that it was asked. if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found { - if cur, err := delivery.DecodeBuilderBuild(v); err == nil { + if cur, err := builds.DecodeBuilderBuild(v); err == nil { switch cur.State { - case delivery.BuildFailed: + case builds.BuildFailed: res.Failed = cur.Error - case delivery.BuildDone: + case builds.BuildDone: default: res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step) } diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index fd9cc4de..fc65815c 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -5,8 +5,10 @@ import ( "fmt" "sort" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) // How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a @@ -24,7 +26,7 @@ type via struct{ Ref, Base string } // carries. A value that is not a candidate is said and left. func (b *Builder) Candidate(ctx context.Context, key string, value []byte) { b.init() - c, err := delivery.DecodeCandidate(value) + c, err := meshcatalog.DecodeCandidatesByGroupValue(value) if err != nil { b.logf("candidates.%s holds what is not a candidate, left: %v", key, err) return @@ -43,7 +45,7 @@ func (b *Builder) ForgetCandidate(key string) { b.mu.Unlock() } -func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) { +func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) { for _, p := range c.Pulls { names := make([]string, 0, len(p.Fps)) for name := range p.Fps { @@ -52,33 +54,33 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) { sort.Strings(names) for _, name := range names { cm := p.Fps[name] - if cm.FP2 == "" || cm.Manifest == "" { + if cm.Fp2 == "" || cm.Manifest == "" { continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by } b.mu.Lock() trunk, known := b.known[name] b.mu.Unlock() if !known || trunk.Path == "" { - b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.FP2) + b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.Fp2) continue } - key, err := delivery.BuilderBuildKey(name, cm.FP2) + key, err := builds.BuilderBuildKey(name, cm.Fp2) if err != nil { b.logf("candidates.%s: %v", c.Group, err) continue } - m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path, - Toolchains: b.agreedFor(ctx, cm.Manifest), Packages: cm.Packages} // the group's package heads (ADR 0573 §4) + m := meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: p.Head, Fp2: cm.Fp2, Manifest: cm.Manifest, Path: trunk.Path, + Toolchains: CatalogToolchains(b.agreedFor(ctx, cm.Manifest)), Packages: cm.Packages} // the group's package heads (ADR 0573 §4) if len(cm.Paths) > 0 { // The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4). - m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths} + m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Paths: cm.Paths} } // The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's // repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the // commit its fingerprint first appeared at, and may be many merges behind the trunk. - base, err := b.trunkHead(ctx, c.Base, p.Repository) + base, err := b.trunkHead(ctx, uint64(c.Base), p.Repository) if err != nil { - b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err) + b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) continue } b.mu.Lock() @@ -94,19 +96,19 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) { // agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the // candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed // is left out, and the build then waits for it as the trunk's does. -func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]delivery.ToolchainUsed { +func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]builds.ToolchainUsed { plan, err, _ := b.planOf(ctx, manifest) if err != nil { return nil } - out := map[string]delivery.ToolchainUsed{} + out := map[string]builds.ToolchainUsed{} for _, bm := range plan.Modules() { if !plan.NeedsToolchain(bm) { continue } language := buildrun.LanguageOf[bm] - if tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)); none == "" { - out[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} + if tc, none := buildstep.AgreedToolchain(b.Toolchains.Entries(language)); none == "" { + out[language] = builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} } } if len(out) == 0 { diff --git a/modules/mesh-delivery-builder/internal/build/candidates_test.go b/modules/mesh-delivery-builder/internal/build/candidates_test.go index 41ca4f10..a20e2c40 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates_test.go +++ b/modules/mesh-delivery-builder/internal/build/candidates_test.go @@ -8,16 +8,20 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/keys" ) // trunkModule writes the trunk's record of the module m at a fingerprint of its own, so that the candidate's fingerprint (fp1) // is one the trunk does not carry. func trunkModule(t *testing.T, r *rig) { t.Helper() - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), - FP2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains}) + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'), + Fp2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}) if err != nil { t.Fatal(err) } @@ -30,13 +34,13 @@ func trunkModule(t *testing.T, r *rig) { func candidateValue(t *testing.T, group string, number int, head string) []byte { t.Helper() - pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: number, Head: head}, Modules: []string{"m"}, - Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a')}}}} - hash, err := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: int64(number), Head: head, Modules: []string{"m"}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a')}}}} + hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) if err != nil { t.Fatal(err) } - data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } @@ -53,7 +57,7 @@ const ( // indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer // internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test). -func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) { +func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c meshcatalog.CandidatesByGroupValue, raw []byte) { t.Helper() data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json") if err != nil { @@ -69,7 +73,7 @@ func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c deli if err := json.Unmarshal(data, &w); err != nil { t.Fatal(err) } - if c, err = delivery.DecodeCandidate(w.Candidate); err != nil { + if c, err = meshcatalog.DecodeCandidatesByGroupValue(w.Candidate); err != nil { t.Fatal(err) } return w.Source.Rev, w.Source.Heads, c, w.Candidate @@ -91,8 +95,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe return heads[repository], nil } const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" - value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'), - Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains}) + value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: p.Repository, Commit: valueCommit, Fp2: "src2:" + hex64('e'), + Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: CatalogToolchains(agreedToolchains)}) if err != nil { t.Fatal(err) } @@ -108,7 +112,7 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches)) } f := fetches[0].body.(map[string]any) - if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head { + if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head { t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev) } } @@ -117,7 +121,7 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 { t.Fatalf("build = %+v, want the candidate's fingerprint built", v) } fetches := r.bm.callsOf("build_go_fetch") @@ -158,7 +162,7 @@ func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { / t.Errorf("fetch args = %+v: a trunk fingerprint is built from the trunk's commit alone", f) } } - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Errorf("build = %+v", v) } } @@ -174,7 +178,7 @@ func TestACandidateWhoseHeadDoesNotMergeIsFailedAndNoTrunkBuildIsStoredUnderIt(t r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 { + if v.State != builds.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 { t.Errorf("%s: build = %+v, builds %d, want failed and nothing built", name, v, len(r.bm.callsOf("build_go_build"))) } } diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index 50b933a9..a45baac7 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -9,8 +9,9 @@ import ( "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/keys" ) // ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given. @@ -37,7 +38,7 @@ type Claimer struct { type Claim struct { Key string Rev uint64 - Build delivery.BuilderBuild + Build builds.BuilderBuild // Gone are the references of a done key the artifact store no longer holds, for which it was claimed again // (ClaimOptions.Stored; issue 1937): said in the log and the key's step. Gone []string @@ -71,7 +72,7 @@ type ClaimOptions struct { // StoredReferences are the references of a done build's bundles and archives that name a blob in the artifact store // (`artifact-store:////blobs/`): what the store's collector can delete. An image or an upstream // copy is a manifest, which the collector keeps, and a package lives in the package registry. -func StoredReferences(b delivery.BuilderBuild) []string { +func StoredReferences(b builds.BuilderBuild) []string { var out []string for _, a := range b.Artifacts { if strings.HasPrefix(a.Reference, "artifact-store://") && strings.Contains(a.Reference, "/blobs/") && @@ -89,12 +90,12 @@ func GoneStep(gone []string) string { // Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has // no reference (a candidate's, ADR 0487 clause 12, ADR 0498 clauses 5 and 6). -func Unpublished(b delivery.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 } +func Unpublished(b builds.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 } // UnpublishedModules are the build modules that publish what a done build holds unpublished: the image build module for // an image, the Node build module for a package; sorted, none when it holds nothing unpublished. -func UnpublishedModules(b delivery.BuilderBuild) []string { - if b.State != delivery.BuildDone { +func UnpublishedModules(b builds.BuilderBuild) []string { + if b.State != builds.BuildDone { return nil } var image, pkg bool @@ -115,7 +116,7 @@ func UnpublishedModules(b delivery.BuilderBuild) []string { // publishable are the build modules of UnpublishedModules that can publish now: the image build module only while a node // runs it (imageBuilder). -func publishable(b delivery.BuilderBuild, imageBuilder func() bool) []string { +func publishable(b builds.BuilderBuild, imageBuilder func() bool) []string { var out []string for _, bm := range UnpublishedModules(b) { if bm != ImageBuildModule || (imageBuilder != nil && imageBuilder()) { @@ -128,8 +129,8 @@ func publishable(b delivery.BuilderBuild, imageBuilder func() bool) []string { // Unreferenced is whether a done build holds a bundle, an archive or an upstream copy that names no reference: a key done // before R4e, whose artifacts stayed on their build nodes (ADR 0498 clause 8). An artifact of no kind (a key written before // kinds) is one too. Claim takes such a key again, once: its new revision names them. -func Unreferenced(b delivery.BuilderBuild) bool { - if b.State != delivery.BuildDone { +func Unreferenced(b builds.BuilderBuild) bool { + if b.State != builds.BuildDone { return false } for _, a := range b.Artifacts { @@ -151,7 +152,7 @@ func Unreferenced(b delivery.BuilderBuild) bool { // - a done key of the trunk's fingerprint whose bundle or archive blob the artifact store no longer holds // (ClaimOptions.Stored, issue 1937) is claimed building as a new revision and built whole, its step saying which; // - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no -// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478 +// builder builds, for an upstream pinned by no digest or a package version taken (builds.FailedUntilChanged: ADR 0478 // clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), // which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for // another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder); @@ -160,11 +161,11 @@ func Unreferenced(b delivery.BuilderBuild) bool { // - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its // own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6). func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string, opt ClaimOptions) (*Claim, error) { - key, err := delivery.BuilderBuildKey(module, fingerprint) + key, err := builds.BuilderBuildKey(module, fingerprint) if err != nil { return nil, err } - building := delivery.BuilderBuild{State: delivery.BuildBuilding, By: c.By, Step: "claimed", Manifest: manifest} + building := builds.BuilderBuild{State: builds.BuildBuilding, By: c.By, Step: "claimed", Manifest: manifest} value, rev, found, err := c.Store.Get(ctx, key) if err != nil { return nil, fmt.Errorf("%s: %w", key, err) @@ -172,12 +173,12 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string if !found { return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Create(ctx, key, data) }) } - cur, err := delivery.DecodeBuilderBuild(value) + cur, err := builds.DecodeBuilderBuild(value) if err != nil { return nil, fmt.Errorf("%s holds what is not a build: %w", key, err) } switch { - case cur.State == delivery.BuildDone: + case cur.State == builds.BuildDone: if Unreferenced(cur) { return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) } @@ -207,13 +208,13 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return &Claim{Key: key, Rev: rev, Build: cur}, nil } return nil, nil - case cur.State == delivery.BuildFailed && (delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): + case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil - case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): + case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): return nil, nil - case cur.State == delivery.BuildFailed: + case cur.State == builds.BuildFailed: return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) - case cur.State == delivery.BuildQueued && cur.Waits != "": + case cur.State == builds.BuildQueued && cur.Waits != "": if opt.Published == nil || !opt.Published(cur.Waits) { return nil, nil // the package it waits for is not published yet } @@ -224,8 +225,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return nil, nil } -func (c Claimer) write(_ context.Context, key string, b delivery.BuilderBuild, put func([]byte) (uint64, error)) (*Claim, error) { - data, err := delivery.EncodeBuilderBuild(b) +func (c Claimer) write(_ context.Context, key string, b builds.BuilderBuild, put func([]byte) (uint64, error)) (*Claim, error) { + data, err := builds.EncodeBuilderBuild(b) if err != nil { return nil, err } @@ -236,18 +237,18 @@ func (c Claimer) write(_ context.Context, key string, b delivery.BuilderBuild, p if err != nil { return nil, fmt.Errorf("%s: %w", key, err) } - b.V = delivery.Version + b.V = keys.Version return &Claim{Key: key, Rev: rev, Build: b}, nil } // Finish moves the claimed key to b by compare-and-set at the claim's revision, and writes nothing when b is the value // the key holds. A key that changed meanwhile is ErrConflict: the claim is no longer this builder's. -func (c Claimer) Finish(ctx context.Context, cl *Claim, b delivery.BuilderBuild) error { - b.V = delivery.Version - if delivery.SameBuilderBuild(cl.Build, b) { +func (c Claimer) Finish(ctx context.Context, cl *Claim, b builds.BuilderBuild) error { + b.V = keys.Version + if builds.SameBuilderBuild(cl.Build, b) { return nil } - data, err := delivery.EncodeBuilderBuild(b) + data, err := builds.EncodeBuilderBuild(b) if err != nil { return err } diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index 54cf5e55..51e93a86 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -6,7 +6,7 @@ import ( "sync" "testing" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // memStore is a bucket's create and compare-and-set in memory, with a revision per write. @@ -78,13 +78,13 @@ const ( digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" ) -func seed(t *testing.T, st *memStore, b delivery.BuilderBuild) string { +func seed(t *testing.T, st *memStore, b builds.BuilderBuild) string { t.Helper() - key, err := delivery.BuilderBuildKey("m", fp1) + key, err := builds.BuilderBuildKey("m", fp1) if err != nil { t.Fatal(err) } - data, err := delivery.EncodeBuilderBuild(b) + data, err := builds.EncodeBuilderBuild(b) if err != nil { t.Fatal(err) } @@ -92,9 +92,9 @@ func seed(t *testing.T, st *memStore, b delivery.BuilderBuild) string { return key } -func valueOf(t *testing.T, st *memStore, key string) delivery.BuilderBuild { +func valueOf(t *testing.T, st *memStore, key string) builds.BuilderBuild { t.Helper() - b, err := delivery.DecodeBuilderBuild(st.vals[key]) + b, err := builds.DecodeBuilderBuild(st.vals[key]) if err != nil { t.Fatal(err) } @@ -108,7 +108,7 @@ func TestClaimCreatesTheKeyBuildingAndASecondBuilderGetsNothing(t *testing.T) { if err != nil || got == nil { t.Fatalf("first claim = %v, %v, want a claim", got, err) } - if v := valueOf(t, st, got.Key); v.State != delivery.BuildBuilding || v.By != "novox.a" || v.Manifest != hex64('a') { + if v := valueOf(t, st, got.Key); v.State != builds.BuildBuilding || v.By != "novox.a" || v.Manifest != hex64('a') { t.Errorf("the key holds %+v, want building by novox.a with the manifest digest", v) } if second, err := b.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}); err != nil || second != nil { @@ -117,7 +117,7 @@ func TestClaimCreatesTheKeyBuildingAndASecondBuilderGetsNothing(t *testing.T) { // A claim losing the create race gets nothing (merged into this test for the test budget, design 51). st = newMem() - st.before = func() { seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.b"}) } + st.before = func() { seed(t, st, builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.b"}) } got, err = Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, "", ClaimOptions{}) if err != nil || got != nil { t.Errorf("claim = %v, %v, want none: the create failed because the key exists", got, err) @@ -126,7 +126,7 @@ func TestClaimCreatesTheKeyBuildingAndASecondBuilderGetsNothing(t *testing.T) { func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3; a key whose artifacts name references (ADR 0498 clause 8) st := newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/x/blobs/" + digest}}}) + seed(t, st, builds.BuilderBuild{State: builds.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []builds.BuiltArtifact{{Name: "x", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/x/blobs/" + digest}}}) writes := st.writes got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}) if err != nil || got != nil || st.writes != writes { @@ -136,25 +136,25 @@ func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3; a key whose arti func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { // 1607/AC3, ADR 0478 clause 2 st := newMem() - key := seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: "the step failed"}) + key := seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "the step failed"}) rev := st.revs[key] got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}) if err != nil || got == nil || st.revs[key] <= rev { t.Fatalf("claim = %v, %v, revision %d (was %d), want a claim as a new revision of the same key", got, err, st.revs[key], rev) } - if v := valueOf(t, st, key); v.State != delivery.BuildBuilding || v.Error != "" { + if v := valueOf(t, st, key); v.State != builds.BuildBuilding || v.Error != "" { t.Errorf("the key holds %+v, want building with no error", v) } st = newMem() - key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: delivery.RefusedKind + "package of language \"python\" (artifact x)"}) + key = seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: builds.RefusedKind + "package of language \"python\" (artifact x)"}) writes := st.writes if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes { t.Errorf("a refused kind: claim = %v, %v, writes %d (was %d), want none and no write", got, err, st.writes, writes) } - for _, e := range []string{delivery.NotPinned + "up: nats:2 names no digest", delivery.VersionTaken + "@novox/mesh-sdk@0.1.15"} { // B3, B5: ADR 0498 clauses 4, 5 + for _, e := range []string{builds.NotPinned + "up: nats:2 names no digest", builds.VersionTaken + "@novox/mesh-sdk@0.1.15"} { // B3, B5: ADR 0498 clauses 4, 5 st = newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: e}) + seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: e}) writes = st.writes if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes { t.Errorf("%s: claim = %v, %v, writes %d (was %d), want none until the fingerprint changes", e, got, err, st.writes, writes) @@ -164,7 +164,7 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { func TestAKeyFailedForNoImageBuilderIsTakenAgainOnlyWhenANodeHasTheModule(t *testing.T) { // 1705b st := newMem() - key := seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: NoImageBuilderPrefix + " no node has mesh-build-image"}) + key := seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: NoImageBuilderPrefix + " no node has mesh-build-image"}) writes := st.writes c := Claimer{Store: st, By: "novox.a"} for _, opt := range []ClaimOptions{{}, {ImageBuilder: func() bool { return false }}} { @@ -173,7 +173,7 @@ func TestAKeyFailedForNoImageBuilderIsTakenAgainOnlyWhenANodeHasTheModule(t *tes } } got, err := c.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{ImageBuilder: func() bool { return true }}) - if err != nil || got == nil || valueOf(t, st, key).State != delivery.BuildBuilding { + if err != nil || got == nil || valueOf(t, st, key).State != builds.BuildBuilding { t.Errorf("claim = %v, %v, want a claim with the key building", got, err) } } @@ -182,11 +182,11 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing st := newMem() c := Claimer{Store: st, By: "novox.a"} got, _ := c.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}) - done := delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Kind: "archive", Digest: digest}}} + done := builds.BuilderBuild{State: builds.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []builds.BuiltArtifact{{Name: "x", Kind: "archive", Digest: digest}}} if err := c.Finish(context.Background(), got, done); err != nil { t.Fatal(err) } - if v := valueOf(t, st, got.Key); v.State != delivery.BuildDone || v.Artifacts[0].Digest != digest { + if v := valueOf(t, st, got.Key); v.State != builds.BuildDone || v.Artifacts[0].Digest != digest { t.Errorf("the key holds %+v", v) } writes := st.writes @@ -202,7 +202,7 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing c = Claimer{Store: st, By: "novox.a"} got, _ = c.Claim(context.Background(), "m", fp1, "", ClaimOptions{}) st.put(got.Key, []byte(`{"v":1,"state":"failed","error":"someone else"}`)) - err := c.Finish(context.Background(), got, delivery.BuilderBuild{State: delivery.BuildFailed, Error: "mine"}) + err := c.Finish(context.Background(), got, builds.BuilderBuild{State: builds.BuildFailed, Error: "mine"}) if !errors.Is(err, ErrConflict) { t.Errorf("Finish = %v, want ErrConflict", err) } diff --git a/modules/mesh-delivery-builder/internal/build/fakes_test.go b/modules/mesh-delivery-builder/internal/build/fakes_test.go index 2b993dbf..bd18bba5 100644 --- a/modules/mesh-delivery-builder/internal/build/fakes_test.go +++ b/modules/mesh-delivery-builder/internal/build/fakes_test.go @@ -10,7 +10,8 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) type call struct { @@ -36,12 +37,12 @@ func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) // fakeSteps holds the steps values by `/`; Wait on a key it does not hold blocks until the context ends. type fakeSteps struct { mu sync.Mutex - steps map[string]delivery.Step + steps map[string]buildstep.Step // during, when set, is called as a wait starts and the function it answers as the wait ends. during func(key string) func() } -func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) { +func (f *fakeSteps) Wait(ctx context.Context, module, key string) (buildstep.Step, error) { f.mu.Lock() s, ok := f.steps[module+"/"+key] during := f.during @@ -53,11 +54,11 @@ func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step return s, nil } <-ctx.Done() - return delivery.Step{}, context.Cause(ctx) + return buildstep.Step{}, context.Cause(ctx) } -func stepAt(node, id, state, reason string, result string) delivery.Step { - s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason} +func stepAt(node, id, state, reason string, result string) buildstep.Step { + s := buildstep.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason} if result != "" { s.Result = json.RawMessage(result) } @@ -67,7 +68,7 @@ func stepAt(node, id, state, reason string, result string) delivery.Step { // presenceValue is a watchman presence value, as its writer encodes it (mesh-sdk go/delivery EncodePresence). func presenceValue(t *testing.T, online bool) []byte { t.Helper() - b, err := delivery.EncodePresence(delivery.Presence{Online: online}) + b, err := json.Marshal(meshwatchdogs.PresenceByNodeValue{Online: online}) if err != nil { t.Fatal(err) } @@ -77,7 +78,7 @@ func presenceValue(t *testing.T, online bool) []byte { // tcValue is a toolchains value, as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain). func tcValue(t *testing.T, language, version string) []byte { t.Helper() - b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()}) + b, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now().UTC()}) if err != nil { t.Fatal(err) } @@ -85,10 +86,10 @@ func tcValue(t *testing.T, language, version string) []byte { } // tcs are the toolchain entries of a language on nodes, as the toolchains set gives them. -func tcs(language, version string, nodes ...string) []delivery.NodeToolchain { - var out []delivery.NodeToolchain +func tcs(language, version string, nodes ...string) []buildstep.NodeToolchain { + var out []buildstep.NodeToolchain for _, n := range nodes { - out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}}) + out = append(out, buildstep.NodeToolchain{Node: n, Toolchain: buildstep.Toolchain{Language: language, Version: version, Recipe: "1"}}) } return out } diff --git a/modules/mesh-delivery-builder/internal/build/fixtures_test.go b/modules/mesh-delivery-builder/internal/build/fixtures_test.go index 467f91d8..cb04934c 100644 --- a/modules/mesh-delivery-builder/internal/build/fixtures_test.go +++ b/modules/mesh-delivery-builder/internal/build/fixtures_test.go @@ -8,8 +8,9 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buildstep" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // result reads a captured step value of mesh-build-image (testdata/captured/README) and returns its result. @@ -19,11 +20,11 @@ func captured(t *testing.T, name string) json.RawMessage { if err != nil { t.Fatal(err) } - var step delivery.Step + var step sdkstep.Step if err := json.Unmarshal(raw, &step); err != nil { t.Fatalf("%s: %v", name, err) } - if step.State != delivery.StepPassed || len(step.Result) == 0 { + if step.State != sdkstep.StepPassed || len(step.Result) == 0 { t.Fatalf("%s: state %q, result %s, want a passed step with a result", name, step.State, step.Result) } return step.Result @@ -43,15 +44,15 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi }, Build: func(context.Context, map[string]any) (any, error) { return json.RawMessage(captured(t, tc.build)), nil }, } - ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskDelivery, Language: delivery.AskImage, Asker: delivery.BuilderModule, - At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish, - Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)}, - Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}} + ask := sdkstep.StepAsk{ID: buildstep.NewID(), Kind: sdkstep.AskDelivery, Language: sdkstep.AskImage, Asker: builds.BuilderModule, + At: time.Now(), Build: &sdkstep.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish, + Fetch: &sdkstep.Fetch{Repository: "novox/mesh-catalog", Commit: strings.Repeat("a", 40)}, + Artifacts: []sdkstep.AskArtifact{{Artifact: "server"}}}} out, err := runAsk(context.Background(), steps, ask) if err != nil { t.Fatalf("%s: %v", tc.build, err) } - a := out.(delivery.BuildAnswer) + a := out.(sdkstep.BuildAnswer) if len(a.Artifacts) != 1 || a.Artifacts[0].Digest == "" || (a.Artifacts[0].Reference != "") != tc.ref { t.Errorf("%s answered %+v, want a digest and a reference only for the trunk", tc.build, a) } diff --git a/modules/mesh-delivery-builder/internal/build/followups_test.go b/modules/mesh-delivery-builder/internal/build/followups_test.go index ce0efd65..ac03518e 100644 --- a/modules/mesh-delivery-builder/internal/build/followups_test.go +++ b/modules/mesh-delivery-builder/internal/build/followups_test.go @@ -5,20 +5,22 @@ import ( "strings" "testing" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // 1660: at start the builder closes a key left building for a fingerprint the module no longer has. func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *testing.T) { // issue 1660 r := newRig(t, goBundle) - stale := seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD}) // m at fp1 - other, _ := delivery.BuilderBuildKey("gone", fp1) - data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD}) + stale := seed(t, r.st, builds.BuilderBuild{State: builds.BuildBuilding, By: "ace.builder", Manifest: manifestD}) // m at fp1 + other, _ := builds.BuilderBuildKey("gone", fp1) + data, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "ace.builder", Manifest: manifestD}) r.st.put(other, data) // The module m has moved on to another fingerprint. - moved := delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), - FP2: "src2:" + hex64('7'), Manifest: manifestD, Path: "modules/m", Toolchains: agreedToolchains} - enc, _ := delivery.EncodeModule(moved) + moved := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'), + Fp2: "src2:" + hex64('7'), Manifest: manifestD, Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)} + enc, _ := meshcatalogclient.EncodeModule(moved) r.b.Module(context.Background(), "m", enc) r.b.Wait() if err := r.b.Sweep(context.Background()); err != nil { @@ -26,7 +28,7 @@ func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *t } for _, k := range []string{stale, other} { v := valueOf(t, r.st, k) - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "no longer") || !strings.Contains(v.Error, "ace.builder") { + if v.State != builds.BuildFailed || !strings.Contains(v.Error, "no longer") || !strings.Contains(v.Error, "ace.builder") { t.Errorf("%s = %+v, want it closed as failed, naming why and who left it", k, v) } } @@ -34,13 +36,13 @@ func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *t func TestSweepLeavesAKeyOfTheModulesCurrentFingerprintForTheBuildToTakeOver(t *testing.T) { // issue 1660 r := newRig(t, goBundle) - seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD}) + seed(t, r.st, builds.BuilderBuild{State: builds.BuildBuilding, By: "ace.builder", Manifest: manifestD}) r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // takes it over and builds it r.b.Wait() if err := r.b.Sweep(context.Background()); err != nil { t.Fatal(err) } - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.By != "novox.builder" { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.By != "novox.builder" { t.Errorf("build = %+v, want done by novox.builder", v) } } diff --git a/modules/mesh-delivery-builder/internal/build/gone_test.go b/modules/mesh-delivery-builder/internal/build/gone_test.go index c114010a..fea331ca 100644 --- a/modules/mesh-delivery-builder/internal/build/gone_test.go +++ b/modules/mesh-delivery-builder/internal/build/gone_test.go @@ -10,7 +10,7 @@ import ( "testing" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // A done key whose blob the artifact store no longer holds is built again (novox/hq issue 1937): on 2026-10-24 the @@ -76,9 +76,9 @@ func (l *logged) has(words ...string) bool { const goneRef = "artifact-store://m/tool/blobs/" + digest -func doneKeyNaming(ref string) delivery.BuilderBuild { - return delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.builder", Manifest: manifestD, Step: "built mesh-build-go on shanks", - Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: ref}}} +func doneKeyNaming(ref string) builds.BuilderBuild { + return builds.BuilderBuild{State: builds.BuildDone, By: "shanks.builder", Manifest: manifestD, Step: "built mesh-build-go on shanks", + Artifacts: []builds.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: ref}}} } // 1937/AC3 (unit, the claim): a done key of the trunk's fingerprint whose blob is gone is claimed building as a new @@ -86,7 +86,7 @@ func doneKeyNaming(ref string) delivery.BuilderBuild { func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) { st := newMem() c := Claimer{Store: st, By: "novox.builder"} - key, _ := delivery.BuilderBuildKey("m", fp1) + key, _ := builds.BuilderBuildKey("m", fp1) rev := putBuild(t, st, key, doneKeyNaming(goneRef)) held := &fakeHeld{gone: map[string]bool{goneRef: true}} @@ -98,11 +98,11 @@ func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) { if err != nil || cl == nil { t.Fatalf("claim = %+v, %v; want the key claimed again", cl, err) } - if cl.Rev <= rev || cl.Build.State != delivery.BuildBuilding || !slices.Equal(cl.Gone, []string{goneRef}) { + if cl.Rev <= rev || cl.Build.State != builds.BuildBuilding || !slices.Equal(cl.Gone, []string{goneRef}) { t.Errorf("claim = %+v (revision was %d); want building as a new revision naming the gone reference", cl, rev) } v := valueOf(t, st, key) - if v.State != delivery.BuildBuilding || !strings.Contains(v.Step, "no longer holds") || !strings.Contains(v.Step, goneRef) { + if v.State != builds.BuildBuilding || !strings.Contains(v.Step, "no longer holds") || !strings.Contains(v.Step, goneRef) { t.Errorf("the key holds %+v; want building, its step naming the gone reference", v) } } @@ -112,9 +112,9 @@ func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) { func TestADoneTrunkKeyStaysDoneWhileTheStoreHoldsItsBlobs(t *testing.T) { st := newMem() c := Claimer{Store: st, By: "novox.builder"} - key, _ := delivery.BuilderBuildKey("m", fp1) + key, _ := builds.BuilderBuildKey("m", fp1) b := doneKeyNaming(goneRef) - b.Artifacts = append(b.Artifacts, delivery.BuiltArtifact{Name: "img", Kind: "image", Digest: digest, Reference: "registry.example/m/img@" + digest}) + b.Artifacts = append(b.Artifacts, builds.BuiltArtifact{Name: "img", Kind: "image", Digest: digest, Reference: "registry.example/m/img@" + digest}) rev := putBuild(t, st, key, b) held := &fakeHeld{gone: map[string]bool{}} if cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}); err != nil || cl != nil { @@ -143,7 +143,7 @@ func TestTheBuilderBuildsAgainADoneKeyWhoseBlobIsGone(t *testing.T) { r.b.Stored = held.stored rev := putBuild(t, r.st, r.key, doneKeyNaming(goneRef)) v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 { + if v.State != builds.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 { t.Fatalf("build = %+v at revision %d (was %d); want built again", v, r.st.revs[r.key], rev) } if !strings.Contains(v.Step, "built again: the artifact store no longer held "+goneRef) { @@ -177,7 +177,7 @@ func TestArtifactsGoneMakesTheBuilderAskAndBuildAgain(t *testing.T) { held := &fakeHeld{gone: map[string]bool{}} r.b.Stored = held.stored putBuild(t, r.st, r.key, doneKeyNaming(goneRef)) - if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone || len(r.bm.callsOf("build_go_build")) != 0 { + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || len(r.bm.callsOf("build_go_build")) != 0 { t.Fatalf("a key whose blob is held was built: %+v", v) } rev := r.st.revs[r.key] @@ -196,7 +196,7 @@ func TestArtifactsGoneMakesTheBuilderAskAndBuildAgain(t *testing.T) { Key: r.key, Fingerprint: fp1, Gone: []string{goneRef}}) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 { + if v.State != builds.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 { t.Fatalf("build = %+v at revision %d (was %d); want built again on the event", v, r.st.revs[r.key], rev) } if !log.has("heard artifacts-gone") || !log.has(r.key, "built again", goneRef) { diff --git a/modules/mesh-delivery-builder/internal/build/imports_test.go b/modules/mesh-delivery-builder/internal/build/imports_test.go index 77215db7..6f81e2fe 100644 --- a/modules/mesh-delivery-builder/internal/build/imports_test.go +++ b/modules/mesh-delivery-builder/internal/build/imports_test.go @@ -6,7 +6,8 @@ import ( "sync" "testing" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) // ADR 0593 §2 (F3): the imports a Go build module answers at a build's end reach the done key and the log's end entry, @@ -14,7 +15,7 @@ import ( func importsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) { r := newRig(t, goBundle) imports := []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-sdk/go/announce"} - r.asks.Answer = func(a delivery.StepAsk) delivery.Step { + r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { step := r.answer(a) var answer map[string]any if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" { @@ -23,11 +24,11 @@ func importsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) { } return step } - var logged []delivery.BuildLogEntry + var logged []builds.BuildLogEntry var mu sync.Mutex - r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() } + r.b.Node, r.b.Log = "novox", func(e builds.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() } v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") { + if v.State != builds.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") { t.Fatalf("done key %+v, want the imports", v) } if len(logged) != 2 || strings.Join(logged[1].Imports, ",") != strings.Join(imports, ",") || len(logged[0].Imports) != 0 { @@ -51,7 +52,7 @@ func outOfStepFilesABuildAnswersReachTheLog(t *testing.T) { } { t.Run(row.name, func(t *testing.T) { r := newRig(t, goBundle) - r.asks.Answer = func(a delivery.StepAsk) delivery.Step { + r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { step := r.answer(a) answer := map[string]any{} _ = json.Unmarshal(step.Result, &answer) @@ -60,13 +61,13 @@ func outOfStepFilesABuildAnswersReachTheLog(t *testing.T) { } step.Result, _ = json.Marshal(answer) if row.fail { - step.State, step.Reason = delivery.StepFailed, "the generated files are out of step" + step.State, step.Reason = buildstep.StepFailed, "the generated files are out of step" } return step } - var logged []delivery.BuildLogEntry + var logged []builds.BuildLogEntry var mu sync.Mutex - r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() } + r.b.Node, r.b.Log = "novox", func(e builds.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() } r.consider(t, "modules/m") mu.Lock() defer mu.Unlock() diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index d8b14932..6cf6250f 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -6,7 +6,9 @@ import ( "strconv" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/trunk" "mesh-delivery-builder/internal/versions" ) @@ -16,8 +18,8 @@ import ( type Numbers interface { // tag is a check's build's tag (TagOf), empty for a trunk build; done the digests of its key when it is done already // (versions.BuildClaim.Done), nil otherwise. - Number(ctx context.Context, module string, m delivery.Module, plan Plan, tag string, done map[string]string) (versions.Numbering, error) - Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error + Number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, tag string, done map[string]string) (versions.Numbering, error) + Built(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, got versions.Numbering, made []builds.BuiltArtifact) error } // VersionNumbers is Numbers over the builder's state versions. @@ -30,11 +32,11 @@ type VersionNumbers struct { // ClaimOf is the claim a module value's trunk build is numbered by: a package per source revision (Src), the rest per // fingerprint, in merge order (Src). -func ClaimOf(module string, m delivery.Module, plan Plan) versions.BuildClaim { - c := versions.BuildClaim{Module: module, Path: m.Path, Repository: delivery.RepositoryName(m.Repository), Fingerprint: m.FP2, - Commit: m.Commit, Merge: m.Src} - if m.Src != 0 { - c.Revision = strconv.FormatUint(m.Src, 10) +func ClaimOf(module string, m meshcatalog.ModulesByModuleValue, plan Plan) versions.BuildClaim { + c := versions.BuildClaim{Module: module, Path: m.Path, Repository: trunk.RepositoryName(m.Repository), Fingerprint: m.Fp2, + Commit: m.Commit, Merge: uint64(m.Src)} + if uint64(m.Src) != 0 { + c.Revision = strconv.FormatUint(uint64(m.Src), 10) } for _, t := range plan.Tasks { c.Artifacts = append(c.Artifacts, versions.ClaimArtifact{Artifact: t.Artifact, Kind: t.Kind, Language: t.Language}) @@ -42,7 +44,7 @@ func ClaimOf(module string, m delivery.Module, plan Plan) versions.BuildClaim { return c } -func (v VersionNumbers) Number(ctx context.Context, module string, m delivery.Module, plan Plan, tag string, done map[string]string) (versions.Numbering, error) { +func (v VersionNumbers) Number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, tag string, done map[string]string) (versions.Numbering, error) { c := ClaimOf(module, m, plan) c.Done = done if tag != "" { @@ -56,7 +58,7 @@ func (v VersionNumbers) Number(ctx context.Context, module string, m delivery.Mo return got, err } -func (v VersionNumbers) Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error { +func (v VersionNumbers) Built(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, got versions.Numbering, made []builds.BuiltArtifact) error { digests := map[string]string{} for _, a := range made { digests[a.Name] = a.Digest @@ -71,32 +73,32 @@ type TagOf func(ctx context.Context, repository, head string) (tag string, ok bo // number numbers a build before its claim, a trunk build's artifacts or a check's build's packages under its tag: false // when it waits for its floor (a package, design 51 *The floor*), and then nothing is claimed; the floor's wake looks at // the module again. -func (b *Builder) number(ctx context.Context, module string, m delivery.Module, plan Plan, tag string) (versions.Numbering, bool) { +func (b *Builder) number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, tag string) (versions.Numbering, bool) { if b.Numbers == nil { return versions.Numbering{}, true } var done map[string]string if tag == "" { - done = b.doneDigests(ctx, module, m.FP2) + done = b.doneDigests(ctx, module, m.Fp2) } got, err := b.Numbers.Number(ctx, module, m, plan, tag, done) for _, s := range got.Said { - b.logf("%s at %s: %s", module, m.FP2, s.Line) + b.logf("%s at %s: %s", module, m.Fp2, s.Line) // And in the build log (design 51 *The versions*: it says a version was kept, and why), between the build's start and end. // A look at a key already done builds nothing and has no start or end: a version kept there says nothing new, and is // not logged (each later look, a first view answered or a name heard, would add one). if b.Log != nil && !(done != nil && s.Why == versions.SaidKept) { - b.Log(delivery.BuildLogEntry{Event: delivery.BuildSaidEvent, Module: module, FP: m.FP2, Commit: m.Commit, Node: b.Node, + b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: time.Now().UTC(), Line: s.Line, Why: s.Why}) } } if err != nil { // Said, and built as before: an unnumbered build is published by its digest (ADR 0617 §1), numbered at its next look. - b.logf("%s at %s: %v", module, m.FP2, fmt.Errorf("not numbered, the versions cannot be used: %w", err)) + b.logf("%s at %s: %v", module, m.Fp2, fmt.Errorf("not numbered, the versions cannot be used: %w", err)) return versions.Numbering{}, true } if got.Wait != "" { - b.logf("%s at %s is not built yet: %s", module, m.FP2, got.Wait) + b.logf("%s at %s is not built yet: %s", module, m.Fp2, got.Wait) return got, false } return got, true @@ -105,7 +107,7 @@ func (b *Builder) number(ctx context.Context, module string, m delivery.Module, // doneDigests are the digests of a trunk build's key by artifact when the key is done, nil otherwise or when it cannot be // read: a look at a done build numbers what it left unnumbered with them (versions.BuildClaim.Done). func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string]string { - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil || b.Claimer.Store == nil { return nil } @@ -113,8 +115,8 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string if err != nil || !found { return nil } - cur, err := delivery.DecodeBuilderBuild(raw) - if err != nil || cur.State != delivery.BuildDone { + cur, err := builds.DecodeBuilderBuild(raw) + if err != nil || cur.State != builds.BuildDone { return nil } out := map[string]string{} diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index 4a5969a1..11c678d5 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -9,8 +9,8 @@ import ( "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that @@ -63,19 +63,19 @@ func (p Plan) NeedsToolchain(module string) bool { } // Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) no build module -// builds. It opens delivery.RefusedKind, so its key is not built again until the fingerprint changes. +// builds. It opens builds.RefusedKind, so its key is not built again until the fingerprint changes. type Refused struct{ Artifact, What string } func (r *Refused) Error() string { - return fmt.Sprintf("%s%s (artifact %s): no build module builds it; the builder builds Go and TypeScript bundles, archives, images, upstream copies and TypeScript packages", delivery.RefusedKind, r.What, r.Artifact) + return fmt.Sprintf("%s%s (artifact %s): no build module builds it; the builder builds Go and TypeScript bundles, archives, images, upstream copies and TypeScript packages", builds.RefusedKind, r.What, r.Artifact) } // NotPinned is the error of a plan with an upstream artifact whose reference names no digest (ADR 0498 clause 4): it opens -// delivery.NotPinned, so its key is not built again until the fingerprint changes. +// builds.NotPinned, so its key is not built again until the fingerprint changes. type NotPinned struct{ Artifact, From string } func (n *NotPinned) Error() string { - return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", delivery.NotPinned, n.Artifact, n.From) + return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", builds.NotPinned, n.Artifact, n.From) } // PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2, ADR 0498 clauses 4 and 5): a Go bundle and an archive are diff --git a/modules/mesh-delivery-builder/internal/build/plan_test.go b/modules/mesh-delivery-builder/internal/build/plan_test.go index 9914ca92..897f3e1c 100644 --- a/modules/mesh-delivery-builder/internal/build/plan_test.go +++ b/modules/mesh-delivery-builder/internal/build/plan_test.go @@ -7,7 +7,7 @@ import ( "testing" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -58,21 +58,21 @@ func TestPlanSendsAPinnedUpstreamToTheImageBuildModulesCopyAndRefusesAnUnpinnedO t.Error("a copy needs no toolchain") } _, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"})) - if err == nil || !strings.HasPrefix(err.Error(), delivery.NotPinned+"up") || !delivery.FailedUntilChanged(err.Error()) { - t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, delivery.NotPinned) + if err == nil || !strings.HasPrefix(err.Error(), builds.NotPinned+"up") || !builds.FailedUntilChanged(err.Error()) { + t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, builds.NotPinned) } } func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 clause 5 _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"})) var r *Refused - if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), delivery.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") { - t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, delivery.RefusedKind) + if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), builds.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") { + t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, builds.RefusedKind) } } func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) { - if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), delivery.RefusedKind) { + if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), builds.RefusedKind) { t.Errorf("a python bundle: error = %v, want a refusal", err) } p, err := PlanOf(manifestOf()) diff --git a/modules/mesh-delivery-builder/internal/build/queue.go b/modules/mesh-delivery-builder/internal/build/queue.go index 7542a503..68d94081 100644 --- a/modules/mesh-delivery-builder/internal/build/queue.go +++ b/modules/mesh-delivery-builder/internal/build/queue.go @@ -6,7 +6,8 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) // **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the @@ -19,7 +20,7 @@ const ( SeatQueueTool = "builder_seat_queue" // QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language. QueuesState = "queues" - QueuesBucket = delivery.BuilderModule + "_" + QueuesState + QueuesBucket = builds.BuilderModule + "_" + QueuesState // DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind. DefaultQueueLimit = 50 MaxQueueLimit = 500 @@ -87,7 +88,7 @@ func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue { // NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value. func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) { - if _, err := delivery.AskStream(language); err != nil { + if _, err := buildstep.AskStream(language); err != nil { return "", nil, fmt.Errorf("language: %w", err) } if limit <= 0 { diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index a9ab4f29..562010ce 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -2,12 +2,17 @@ package build import ( "context" + "encoding/json" "strings" "testing" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/keys" ) // R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait @@ -19,7 +24,7 @@ func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testi r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone { + if v.State != builds.BuildDone { t.Fatalf("build = %+v, want done", v) } if ok, missing := v.Referenced(); !ok { @@ -40,21 +45,21 @@ func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *tes r := newRig(t, goBundle) r.bm.noRef["ace/tool"] = true v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "tool") { - t.Fatalf("build = %+v, want failed opening %q and naming the artifact", v, delivery.NotStored) + if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotStored) || !strings.Contains(v.Error, "tool") { + t.Fatalf("build = %+v, want failed opening %q and naming the artifact", v, builds.NotStored) } delete(r.bm.noRef, "ace/tool") - if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + if v = r.consider(t, "modules/m"); v.State != builds.BuildDone { t.Errorf("build = %+v, want a not-stored key built again as a new revision", v) } } func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1, ADR 0498 clause 2 r := newRig(t, goBundle) - r.bm.fails["ace/build_go_build"] = delivery.NotStored + "tool: the artifact store answered 507 insufficient storage" + r.bm.fails["ace/build_go_build"] = builds.NotStored + "tool: the artifact store answered 507 insufficient storage" v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || !strings.Contains(v.Error, "507") || !strings.Contains(v.Error, "ace") { - t.Fatalf("build = %+v, want failed opening %q with the store's words and the node", v, delivery.NotStored) + if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotStored) || !strings.Contains(v.Error, "507") || !strings.Contains(v.Error, "ace") { + t.Fatalf("build = %+v, want failed opening %q with the store's words and the node", v, builds.NotStored) } } @@ -78,9 +83,9 @@ func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's } // putBuild writes a builder key's value as the store would hold it. -func putBuild(t *testing.T, st *memStore, key string, b delivery.BuilderBuild) uint64 { +func putBuild(t *testing.T, st *memStore, key string, b builds.BuilderBuild) uint64 { t.Helper() - data, err := delivery.EncodeBuilderBuild(b) + data, err := builds.EncodeBuilderBuild(b) if err != nil { t.Fatal(err) } @@ -91,10 +96,10 @@ func putBuild(t *testing.T, st *memStore, key string, b delivery.BuilderBuild) u func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { // B2, ADR 0498 clause 8 r := newRig(t, goBundle, archiveA) - rev := putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "built mesh-build-go on ace and novox", - Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest}, {Name: "tool", Kind: "bundle", Digest: digest}}}) + rev := putBuild(t, r.st, r.key, builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "built mesh-build-go on ace and novox", + Artifacts: []builds.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest}, {Name: "tool", Kind: "bundle", Digest: digest}}}) v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev { + if v.State != builds.BuildDone || r.st.revs[r.key] <= rev { t.Fatalf("build = %+v at revision %d (was %d), want built again as a new revision", v, r.st.revs[r.key], rev) } if ok, missing := v.Referenced(); !ok { @@ -113,7 +118,7 @@ func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheRe r := newRig(t, up) r.puller = "ace" v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 { t.Fatalf("build = %+v, want done with the copy", v) } if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest { @@ -146,13 +151,13 @@ func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode( func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2 r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) r.puller = "ace" - r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500" + r.bm.fails["ace/build_image_copy"] = builds.NotStored + "up: the registry answered 500" v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) { + if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotStored) || builds.FailedUntilChanged(v.Error) { t.Fatalf("build = %+v, want failed 'not stored:' and built again", v) } delete(r.bm.fails, "ace/build_image_copy") - if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + if v = r.consider(t, "modules/m"); v.State != builds.BuildDone { t.Errorf("build = %+v, want done on the new revision", v) } } @@ -166,7 +171,7 @@ func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t * c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) c.b.Wait() v := valueOf(t, c.st, c.key) - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" { t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v) } for _, call := range c.bm.callsOf("build_node_build") { @@ -178,7 +183,7 @@ func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t * r := newRig(t, tsPackage) r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" v = r.consider(t, "modules/m") - if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" { + if v.State != builds.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" { t.Fatalf("trunk build = %+v, want done naming the published version", v) } if n := len(r.bm.callsOf("build_node_build")); n == 0 { @@ -189,11 +194,11 @@ func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t * func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B5, ADR 0498 clause 5 r := newRig(t, tsPackage) r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" - r.bm.fails["ace/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" + r.bm.fails["ace/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" r.bm.fails["novox/build_node_build"] = r.bm.fails["ace/build_node_build"] v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) || !strings.Contains(v.Error, "0.1.15") { - t.Fatalf("build = %+v, want failed opening %q naming the version", v, delivery.VersionTaken) + if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) || !strings.Contains(v.Error, "0.1.15") { + t.Fatalf("build = %+v, want failed opening %q naming the version", v, builds.VersionTaken) } writes := r.st.writes r.consider(t, "modules/m") @@ -208,7 +213,7 @@ func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *tes trunkModule(t, r) r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Fatalf("candidate build = %+v", v) } rev := r.st.revs[r.key] @@ -237,12 +242,12 @@ func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() for _, n := range []string{"ace", "novox"} { - r.bm.fails[n+"/build_node_build"] = delivery.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" + r.bm.fails[n+"/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file" } r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.VersionTaken) { - t.Errorf("build = %+v, want failed opening %q", v, delivery.VersionTaken) + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) { + t.Errorf("build = %+v, want failed opening %q", v, builds.VersionTaken) } } @@ -251,8 +256,8 @@ func sdkModule(t *testing.T, r *rig, ref string) []byte { t.Helper() r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{tsPackage}}} r.bm.packages["lib"] = ref - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "8a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: "src2:" + hex64('5'), - Manifest: mdigest('b'), Path: ".", Toolchains: agreedToolchains}) + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: "8a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: "src2:" + hex64('5'), + Manifest: mdigest('b'), Path: ".", Toolchains: CatalogToolchains(agreedToolchains)}) if err != nil { t.Fatal(err) } @@ -266,7 +271,7 @@ func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePubli r.bm.fails[n+"/build_node_fetch"] = waitsFor + ref + ": the lock names a version the registry does not hold" } v := r.consider(t, "modules/m") - if v.State != delivery.BuildQueued || v.Waits != ref || v.Error != "" { + if v.State != builds.BuildQueued || v.Waits != ref || v.Error != "" { t.Fatalf("build = %+v, want queued, waiting for %s, not failed", v, ref) } delete(r.bm.fails, "ace/build_node_fetch") @@ -279,7 +284,7 @@ func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePubli } r.b.Module(context.Background(), "sdk", sdkModule(t, r, ref)) // the package's trunk key is built and published r.b.Wait() - if v = valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Waits != "" { + if v = valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Waits != "" { t.Errorf("build = %+v, want built once the package it waits for is published", v) } } @@ -287,16 +292,16 @@ func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePubli func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *testing.T) { // B7: the scan at start const ref = "@novox/mesh-sdk@0.1.15" r := newRig(t, tsBundle) - putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildQueued, By: "novox.builder", Manifest: manifestD, Waits: ref, Step: "waits"}) - sdkKey, _ := delivery.BuilderBuildKey("sdk", "src2:"+hex64('5')) - putBuild(t, r.st, sdkKey, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: mdigest('b'), Step: "built", - Artifacts: []delivery.BuiltArtifact{{Name: "lib", Kind: "package", Digest: digest, Reference: ref}}}) + putBuild(t, r.st, r.key, builds.BuilderBuild{State: builds.BuildQueued, By: "novox.builder", Manifest: manifestD, Waits: ref, Step: "waits"}) + sdkKey, _ := builds.BuilderBuildKey("sdk", "src2:"+hex64('5')) + putBuild(t, r.st, sdkKey, builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Manifest: mdigest('b'), Step: "built", + Artifacts: []builds.BuiltArtifact{{Name: "lib", Kind: "package", Digest: digest, Reference: ref}}}) r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) if err := r.b.Sweep(context.Background()); err != nil { t.Fatal(err) } r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Errorf("build = %+v, want built: the package it waits for was published before the builder started", v) } } @@ -307,9 +312,9 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')} // The named tree the indexer computes from those paths (its own, ADR 0593 §2), given as data: the builder reads it, // never computes it. - const tree = delivery.NamedTreePrefix + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281" - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".", - Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}}) + const tree = "named:" + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281" + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: manifestD, Path: ".", + Toolchains: CatalogToolchains(agreedToolchains), Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: tree, Paths: paths}}) if err != nil { t.Fatal(err) } @@ -335,7 +340,7 @@ func hex40(c byte) string { return strings.Repeat(string(c), 40) } func TestOneNodeBuildsEachFingerprint(t *testing.T) { // B9, ADR 0478 clause 4: "From R4e one node builds each fingerprint" r := newRig(t, goBundle, archiveA, tsBundle) // ace and novox both qualify for both build modules v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || strings.Contains(v.Step, " and ") || strings.HasPrefix(v.Step, "one node (") { + if v.State != builds.BuildDone || strings.Contains(v.Step, " and ") || strings.HasPrefix(v.Step, "one node (") { t.Fatalf("build = %+v, want done on one node of each build module, unmarked", v) } if f, b := len(r.bm.callsOf("build_go_fetch")), len(r.bm.callsOf("build_go_build")); f != 1 || b != 2 { @@ -348,9 +353,9 @@ func TestOneNodeBuildsEachFingerprint(t *testing.T) { // B9, ADR 0478 clause 4: func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: the second-node retake is gone r := newRig(t, goBundle) - putBuild(t, r.st, r.key, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.builder", Manifest: manifestD, + putBuild(t, r.st, r.key, builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "one node (mesh-build-go): built mesh-build-go on one node, novox", - Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/tool/blobs/" + digest}}}) + Artifacts: []builds.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/tool/blobs/" + digest}}}) writes := r.st.writes r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.NodesChanged(context.Background()) @@ -364,10 +369,10 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A r := newRig(t, tsPackage) r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15" trunkModule(t, r) - pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: headX}, Modules: []string{"m"}, - Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}} - hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) - data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m"}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}} + hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) + data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index f09fa154..8d7d8a1d 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -6,13 +6,15 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" ) -func haveValue(t *testing.T, phase delivery.Phase, health delivery.Health) []byte { +func haveValue(t *testing.T, phase, health string) []byte { t.Helper() - b, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: health}) + b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Phase: phase, Health: health}) if err != nil { t.Fatal(err) } @@ -23,7 +25,7 @@ func haveValue(t *testing.T, phase delivery.Phase, health delivery.Health) []byt // fingerprint names no toolchain, and the builder builds it on the build module's nodes with no toolchain agreed. func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *testing.T) { // review round 4, 1 r := newRig(t, archiveA) - v := func() delivery.BuilderBuild { + v := func() builds.BuilderBuild { r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) r.b.Wait() if _, ok := r.st.vals[r.key]; !ok { @@ -31,7 +33,7 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes } return valueOf(t, r.st, r.key) }() - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 { t.Fatalf("build = %+v, want done with the archive and no toolchain named", v) } if n := len(r.bm.callsOf("build_go_build")); n != 1 { @@ -52,7 +54,7 @@ func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testi func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2 st := newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) + seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) writes := st.writes got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true}) if err != nil || got != nil || st.writes != writes { @@ -63,7 +65,7 @@ func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review // 6. A key found building is taken over, whoever holds it. func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6 st := newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.a"}) + seed(t, st, builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.a"}) if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got == nil { t.Errorf("own building key: claim = %v, %v, want it taken again", got, err) } @@ -77,9 +79,9 @@ func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // rev func TestABuilderFindingAKeyBuildingByAnotherNodeBuildsItOnStart(t *testing.T) { // review round 4, 6 r := newRig(t, goBundle) - seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD}) + seed(t, r.st, builds.BuilderBuild{State: builds.BuildBuilding, By: "ace.builder", Manifest: manifestD}) v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || v.By != "novox.builder" { + if v.State != builds.BuildDone || v.By != "novox.builder" { t.Errorf("build = %+v, want it taken over and done by novox.builder", v) } } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e651e436..50271829 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -13,19 +13,23 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/forms" + repotrunk "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-builder/internal/versions" ) // ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). type ToolchainEntries interface { - Entries(language string) []delivery.NodeToolchain + Entries(language string) []sdkstep.NodeToolchain } // Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is @@ -38,14 +42,14 @@ type Builder struct { // Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq // ADR 0525 §1, §3, §6; issue 1887 A4): the builder chooses no node, whichever build node pulls the ask builds it. Asks interface { - Put(ctx context.Context, a delivery.StepAsk) error - Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) + Put(ctx context.Context, a sdkstep.StepAsk) error + Wait(ctx context.Context, a sdkstep.StepAsk) (sdkstep.Step, error) } // Held says whether an ask a key names is on its stream or a build node wrote a step for it (issue 1910): a key taken // over at start whose ask is neither was left by a builder that stopped between naming the ask and putting it, and // the ask is put then, under the same id. Nil takes every named ask as held. Held interface { - Held(ctx context.Context, a delivery.StepAsk) (bool, error) + Held(ctx context.Context, a sdkstep.StepAsk) (bool, error) } Toolchains ToolchainEntries Manifests Manifests @@ -66,7 +70,7 @@ type Builder struct { // ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked // again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it. Gone interface { - Gone(ctx context.Context, a delivery.StepAsk) (string, error) + Gone(ctx context.Context, a sdkstep.StepAsk) (string, error) } CheckAfter time.Duration // Askings is how many times one build module's part of a build is asked, the first included, before its key is failed @@ -78,7 +82,7 @@ type Builder struct { // Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody. Said func(AskedAgain) // Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody. - Log func(delivery.BuildLogEntry) + Log func(builds.BuildLogEntry) // Now is the builder's clock: of the log, and of the decline bound (DeclineBound runs from the first decline); nil is // time.Now. Node is the node the builder runs on, which asked the build. Now func() time.Time @@ -98,12 +102,12 @@ type Builder struct { sem chan struct{} wg sync.WaitGroup mu sync.Mutex - known map[string]delivery.Module // by modules key - inflight map[string]bool // by build key - again map[string]delivery.Module // by build key: the newest value that arrived while the key was being built - plans map[string]planned // by manifest digest - cands map[string]delivery.Candidate // by candidates key (candidates.go) - via map[string]via // by build key: a candidate's fetch (candidates.go) + known map[string]meshcatalog.ModulesByModuleValue // by modules key + inflight map[string]bool // by build key + again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built + plans map[string]planned // by manifest digest + cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go) + via map[string]via // by build key: a candidate's fetch (candidates.go) // outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them. outOfStep map[string][]string trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint @@ -132,8 +136,8 @@ func (b *Builder) init() { limit = 256 } b.sem = make(chan struct{}, limit) - b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} - b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} + b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} + b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands = map[string]*handWatch{} }) @@ -152,14 +156,14 @@ func (b *Builder) Wait() { b.wg.Wait() } // to be built. A value that is not a module is said and left. func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.init() - m, err := delivery.DecodeModule(value) + m, err := meshcatalogclient.DecodeModule(value) if err != nil { b.logf("modules.%s holds what is not a module, left: %v", key, err) return } b.mu.Lock() b.known[key] = m - if bk, err := delivery.BuilderBuildKey(key, m.FP2); err == nil && m.FP2 != "" { + if bk, err := builds.BuilderBuildKey(key, m.Fp2); err == nil && m.Fp2 != "" { // A fingerprint the trunk carries is built from the trunk's commit, whoever asked for it first. b.trunkFP[bk] = true delete(b.via, bk) @@ -185,8 +189,8 @@ func (b *Builder) Sweep(ctx context.Context) error { b.mu.Lock() current := map[string]bool{} for name, m := range b.known { - if m.FP2 != "" { - if k, err := delivery.BuilderBuildKey(name, m.FP2); err == nil { + if m.Fp2 != "" { + if k, err := builds.BuilderBuildKey(name, m.Fp2); err == nil { current[k] = true } } @@ -198,17 +202,17 @@ func (b *Builder) Sweep(ctx context.Context) error { if err != nil || !found { continue } - cur, err := delivery.DecodeBuilderBuild(value) + cur, err := builds.DecodeBuilderBuild(value) if err != nil { continue } newly = b.notePublished(cur) || newly - if current[key] || cur.State != delivery.BuildBuilding { + if current[key] || cur.State != builds.BuildBuilding { continue } - closed := delivery.BuilderBuild{State: delivery.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step, + closed := builds.BuilderBuild{State: builds.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step, Error: fmt.Sprintf("closed at start: left building by %s, and the module no longer has this fingerprint, so nothing will build it", cur.By)} - data, err := delivery.EncodeBuilderBuild(closed) + data, err := builds.EncodeBuilderBuild(closed) if err != nil { return err } @@ -225,8 +229,8 @@ func (b *Builder) Sweep(ctx context.Context) error { } // notePublished keeps the package versions a done build names as references, and says whether one was new. -func (b *Builder) notePublished(v delivery.BuilderBuild) bool { - if v.State != delivery.BuildDone { +func (b *Builder) notePublished(v builds.BuilderBuild) bool { + if v.State != builds.BuildDone { return false } b.mu.Lock() @@ -286,7 +290,7 @@ func (b *Builder) ArtifactsGone(ctx context.Context, e sweep.ArtifactsGone) { b.init() b.mu.Lock() m, ok := b.known[e.Module] - if key, err := delivery.BuilderBuildKey(e.Module, e.Fingerprint); err == nil { + if key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint); err == nil { for id := range b.held { if strings.HasPrefix(id, key+" ") { delete(b.held, id) @@ -294,7 +298,7 @@ func (b *Builder) ArtifactsGone(ctx context.Context, e sweep.ArtifactsGone) { } } b.mu.Unlock() - if !ok || m.FP2 != e.Fingerprint { + if !ok || m.Fp2 != e.Fingerprint { b.logf("%s: heard %s for %s, a fingerprint the module does not have now: nothing to build", e.Module, sweep.EventArtifactsGone, e.Fingerprint) return @@ -308,7 +312,7 @@ func (b *Builder) ArtifactsGone(ctx context.Context, e sweep.ArtifactsGone) { func (b *Builder) Reconsider(ctx context.Context) { b.init() b.mu.Lock() - known := make(map[string]delivery.Module, len(b.known)) + known := make(map[string]meshcatalog.ModulesByModuleValue, len(b.known)) for k, m := range b.known { known[k] = m } @@ -324,7 +328,7 @@ func (b *Builder) Reconsider(ctx context.Context) { func (b *Builder) TrunkMoved(ctx context.Context) { b.init() b.mu.Lock() - cands := make([]delivery.Candidate, 0, len(b.cands)) + cands := make([]meshcatalog.CandidatesByGroupValue, 0, len(b.cands)) for _, c := range b.cands { cands = append(cands, c) } @@ -335,7 +339,7 @@ func (b *Builder) TrunkMoved(ctx context.Context) { } } -func sortedKeys(m map[string]delivery.Module) []string { +func sortedKeys(m map[string]meshcatalog.ModulesByModuleValue) []string { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) @@ -364,11 +368,11 @@ func (b *Builder) planOf(ctx context.Context, digest string) (Plan, error, bool) return plan, err, true } -func (b *Builder) consider(ctx context.Context, module string, m delivery.Module) { - if m.FP2 == "" || m.Manifest == "" { +func (b *Builder) consider(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) { + if m.Fp2 == "" || m.Manifest == "" { return } - key, err := delivery.BuilderBuildKey(module, m.FP2) + key, err := builds.BuilderBuildKey(module, m.Fp2) if err != nil { b.logf("modules.%s: %v", module, err) return @@ -404,7 +408,7 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module // startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and // a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does. -func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) { +func (b *Builder) startOnce(ctx context.Context, key, module string, m meshcatalog.ModulesByModuleValue) { b.wg.Add(1) go func() { defer b.wg.Done() @@ -428,8 +432,8 @@ func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery. // try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left // building by this builder: it takes it again when it starts (ADR 0478 clause 5). -func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { - key, _ := delivery.BuilderBuildKey(module, m.FP2) +func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) { + key, _ := builds.BuilderBuildKey(module, m.Fp2) defer b.handEnded(key) plan, planErr, read := b.planOf(ctx, m.Manifest) nothing := read && planErr == nil && len(plan.Tasks) == 0 @@ -437,7 +441,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if why := b.stale(m, plan); why != "" { // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build // now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written. - b.logf("%s at %s is not built: %s", module, m.FP2, why) + b.logf("%s at %s is not built: %s", module, m.Fp2, why) return } } @@ -448,8 +452,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { b.mu.Unlock() var numbering versions.Numbering tag := "" - if !trunk && b.Tags != nil && b.refOf(module, m.FP2) != "" { - tag, _ = b.Tags(ctx, delivery.RepositoryName(m.Repository), m.Commit) + if !trunk && b.Tags != nil && b.refOf(module, m.Fp2) != "" { + tag, _ = b.Tags(ctx, repotrunk.RepositoryName(m.Repository), m.Commit) } if (trunk || tag != "") && planErr == nil && !nothing { var goOn bool @@ -457,7 +461,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } } - cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, + cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { b.logf("%s: %v", module, err) @@ -467,7 +471,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } b.handClaimed(key, cl.Rev) - b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now())) + b.logBuild(LogStarted(module, m, b.refOf(module, m.Fp2), b.Node, b.now())) if cl.Unasked != nil { b.logf("%v", cl.Unasked) } @@ -477,10 +481,10 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } // A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the // other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6). - var prior *delivery.BuilderBuild + var prior *builds.BuilderBuild var only map[string]bool publishing := false - if cl.Build.State == delivery.BuildDone { + if cl.Build.State == builds.BuildDone { prior, only = &cl.Build, map[string]bool{} if trunk { for _, bm := range publishable(cl.Build, imageSeat) { @@ -489,11 +493,11 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } } } - var result delivery.BuilderBuild + var result builds.BuilderBuild if nothing { // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). - result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"} + result = builds.BuilderBuild{State: builds.BuildDone, Step: "nothing to build: the manifest names no artifact"} } else { result = b.build(ctx, module, m, plan, planErr, prior, only, b.notes(cl)) } @@ -505,10 +509,10 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } result.By, result.Manifest = b.Claimer.By, m.Manifest - if len(cl.Gone) > 0 && result.State == delivery.BuildDone { + if len(cl.Gone) > 0 && result.State == builds.BuildDone { result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")" } - if publishing && result.State == delivery.BuildFailed { + if publishing && result.State == builds.BuildFailed { // The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the // checker and the failed-key retake see it, and a look does not build the image again for nothing. result.Error = keyError([]string{"publishing what the merged commit built failed: " + result.Error}) @@ -517,18 +521,18 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { b.logf("%s: the build could not be written: %v", cl.Key, err) return } - ended := LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result) + ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result) ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26) b.logBuild(ended) - if result.State == delivery.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 { + if result.State == builds.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 { if err := b.Numbers.Built(ctx, module, m, plan, numbering, result.Artifacts); err != nil { b.logf("%s: %v", cl.Key, err) } } - if result.State == delivery.BuildFailed { + if result.State == builds.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } - if result.State == delivery.BuildQueued { + if result.State == builds.BuildQueued { b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits) } if b.notePublished(result) { @@ -539,13 +543,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { // stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan // has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains). -func (b *Builder) stale(m delivery.Module, plan Plan) string { +func (b *Builder) stale(m meshcatalog.ModulesByModuleValue, plan Plan) string { for _, bm := range plan.Modules() { if !plan.NeedsToolchain(bm) { continue // an archive is packed with no toolchain } language := buildrun.LanguageOf[bm] - ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language)) + ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language)) if ag.Wait != "" { return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait) } @@ -558,15 +562,15 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string { return "" } -func failed(format string, args ...any) delivery.BuilderBuild { - return delivery.BuilderBuild{State: delivery.BuildFailed, Error: fmt.Sprintf(format, args...)} +func failed(format string, args ...any) builds.BuilderBuild { + return builds.BuilderBuild{State: builds.BuildFailed, Error: fmt.Sprintf(format, args...)} } // lane is the build of a plan's tasks of one build module on one node. type lane struct { module, node string // node is the one that pulled the lane's ask tasks []Task - toolchain *delivery.ToolchainUsed + toolchain *builds.ToolchainUsed digests map[string]string // artifact -> digest references map[string]string // artifact -> the reference an image was pushed to (a trunk build only) imports []string // the mesh packages the lane's programs import, as its build module answered them @@ -574,7 +578,7 @@ type lane struct { err error } -func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error, prior *delivery.BuilderBuild, only map[string]bool, asked *askNotes) delivery.BuilderBuild { +func (b *Builder) build(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, planErr error, prior *builds.BuilderBuild, only map[string]bool, asked *askNotes) builds.BuilderBuild { switch { case planErr != nil: return failed("%s", planErr) @@ -582,9 +586,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p // Empty is not known (a root module's path is "."): the indexer writes it again (issue 1615). return failed("the module's key names no directory (path), so the build tools cannot be told where %s lives", module) } - repository := delivery.RepositoryName(m.Repository) + repository := repotrunk.RepositoryName(m.Repository) var lanes []*lane - used := map[string]delivery.ToolchainUsed{} + used := map[string]builds.ToolchainUsed{} var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2) var kept []string // build modules whose artifacts stay as the prior done key has them for _, bm := range plan.Modules() { @@ -592,20 +596,20 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p kept = append(kept, bm) continue } - var toolchain *delivery.ToolchainUsed + var toolchain *builds.ToolchainUsed if bm != ImageBuildModule && plan.NeedsToolchain(bm) { // The ask names the toolchain the fingerprint was computed with; a node building with another refuses it. language := buildrun.LanguageOf[bm] - ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language)) + ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language)) if ag.Wait != "" { return failed("the %s toolchain is not agreed: %s", language, ag.Wait) } tc := ag.Toolchain sitting = append(sitting, SittingOutWords(language, ag)...) if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe { - return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written + return builds.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written } - u := delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} + u := builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} used[language], toolchain = u, &u } var tasks []Task @@ -617,7 +621,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p // One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3). lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}}) } - if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { + if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { b.handLanes(key, len(lanes)) } var wg sync.WaitGroup @@ -626,7 +630,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module, asked) }() } wg.Wait() - if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { + if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { b.keepOutOfStep(key, lanes) } var errs []string @@ -639,11 +643,11 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if ref := waitsOf(errs); ref != "" { // The module's lock names a version of a package the registry does not hold yet: the key waits for it, queued, and // is not failed (ADR 0500 clause 6). - return delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref, Step: "waits for the package " + ref + " to be published"} + return builds.BuilderBuild{State: builds.BuildQueued, Waits: ref, Step: "waits for the package " + ref + " to be published"} } return failed("%s", keyError(errs)) } - var artifacts []delivery.BuiltArtifact + var artifacts []builds.BuiltArtifact var present, counted map[string][]string if b.Forms != nil { present, counted = b.Forms() @@ -675,7 +679,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if err != nil { return failed("%s", err) } - built := delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d} + built := builds.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d} var answered []string for _, l := range lanes { if l.module != t.Module { @@ -685,7 +689,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if r == "" && stored(t.Kind) { // A bundle, an archive or an upstream copy is done only when its push was answered with where it is (ADR 0498 // clause 1); a candidate's included (clause 3). - return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", delivery.NotStored, t.Artifact, l.node) + return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", builds.NotStored, t.Artifact, l.node) } if r != "" { answered = append(answered, r) @@ -713,11 +717,11 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if len(used) == 0 { used = nil } - done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, + done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) - if _, err := delivery.EncodeBuilderBuild(done); err != nil { + if _, err := builds.EncodeBuilderBuild(done); err != nil { // A reference the build module answered that does not name the artifact and its digest is no reference. - return failed("%s%v", delivery.NotStored, err) + return failed("%s%v", builds.NotStored, err) } return done } @@ -730,7 +734,7 @@ func stored(kind string) bool { // waitsFor opens the reason of a fetch step whose module's lock names a package version the registry does not hold, then // `@` (the Node build module's, ADR 0500 clause 6). -const waitsFor = delivery.WaitsFor +const waitsFor = builds.WaitsFor // waitsOf is the package version a lane's error says the module waits for, or "" when none does or what it names is not // `@`. @@ -742,7 +746,7 @@ func waitsOf(errs []string) string { } ref, _, _ := strings.Cut(e[at+len(waitsFor):], " ") ref = strings.TrimRight(ref, ":;,") - if _, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildQueued, Waits: ref}); err == nil { + if _, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildQueued, Waits: ref}); err == nil { return ref } } @@ -750,8 +754,8 @@ func waitsOf(errs []string) string { } // keyPrefixes are the errors a build module's step may open its reason with that the key's error must open with too, so -// the claim reads them (delivery.FailedUntilChanged), first the ones not built again. -var keyPrefixes = []string{delivery.NotPinned, delivery.VersionTaken, delivery.RefusedKind, delivery.NotStored} +// the claim reads them (builds.FailedUntilChanged), first the ones not built again. +var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.RefusedKind, builds.NotStored} // keyError is the error of a key from its lanes' errors: when one names a prefix of keyPrefixes inside the step's words // (` on : the step failed: not stored: ...`), the key's error opens with that prefix, so a reader of the @@ -775,8 +779,8 @@ func keyError(errs []string) string { // want of an image builder before R4g is taken again (ADR 0525 §1). func imageSeat() bool { return true } -// askLanguage is the build seat language of each build module (delivery.AskSubject). -var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode, ImageBuildModule: delivery.AskImage} +// askLanguage is the build seat language of each build module (sdkstep.AskSubject). +var askLanguage = map[string]string{buildrun.GoBuildModule: sdkstep.AskGo, buildrun.NodeBuildModule: sdkstep.AskNode, ImageBuildModule: sdkstep.AskImage} // askNotes writes a claimed key's asks into it, building, before each is put (ADR 0525 §1; issue 1887 A4b), and holds // the asks a key taken over at start already names, so they are waited for and never put again. @@ -791,7 +795,7 @@ type askNotes struct { func (b *Builder) notes(cl *Claim) *askNotes { n := &askNotes{b: b, cl: cl} - if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 { + if cl.Build.State == builds.BuildBuilding && len(cl.Build.Asks) > 0 { n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step } return n @@ -809,7 +813,7 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error { n.mu.Lock() defer n.mu.Unlock() v := n.cl.Build - v.State, v.By, v.Step, v.Error, v.Artifacts, v.Needs = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil, nil + v.State, v.By, v.Step, v.Error, v.Artifacts, v.Needs = builds.BuildBuilding, n.b.Claimer.By, step, "", nil, nil v.Asks = maps.Clone(v.Asks) if v.Asks == nil { v.Asks = map[string]string{} @@ -826,9 +830,9 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error { // check stream for a candidate's) and reads what the node that pulled it built (ADR 0525 §1, §3, §6). An upstream artifact // is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches nothing. A lane of a // key taken over at start waits for the ask the key names and puts none. -func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string, asked *askNotes) { - req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit} - if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { +func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) { + req := sdkstep.Fetch{Repository: repository, Commit: m.Commit} + if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { b.mu.Lock() v, candidate := b.via[key] b.mu.Unlock() @@ -847,26 +851,26 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del } candidate := req.Base != "" language := askLanguage[l.module] - kind := delivery.AskDelivery + kind := sdkstep.AskDelivery if candidate { - kind = delivery.AskCheck + kind = sdkstep.AskCheck } - job := &delivery.BuildJob{Module: module, Path: m.Path, FP: m.FP2, Toolchain: l.toolchain, + job := &sdkstep.BuildJob{Module: module, Path: m.Path, Fp: m.Fp2, Toolchain: buildToolchain(language, l.toolchain), // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). Publish: !candidate} for _, t := range l.tasks { - a := delivery.AskArtifact{Artifact: t.Artifact} + a := sdkstep.AskArtifact{Artifact: t.Artifact} if t.Kind == catalogue.ArtifactUpstream { a.From = t.From // the copy runs no code of a change: the module's name, the artifact and its pin } job.Artifacts = append(job.Artifacts, a) } if needsFetch(l.tasks) { - job.Fetch = req.Args() + job.Fetch = &req } - ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job} - if language == delivery.AskGo && job.Fetch != nil && len(m.Packages) > 0 { + ask := sdkstep.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: builds.BuilderModule, At: time.Now(), Build: job} + if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 { // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. ask.Packages = maps.Clone(m.Packages) @@ -899,29 +903,29 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } } - step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2) + step, ask, err := b.answer(ctx, l, ask, asked, module, m.Fp2) if err != nil { l.err = fmt.Errorf("%s: %w", l.module, err) return } l.node = step.Node // The build module's compare of a contracts module's generated files (design 26, CAT-BG5/BN2) answers the files out - // of step as delivery.BuildAnswer.OutOfStep, in a failed step's result too: read before the step's state is. - var compared delivery.BuildAnswer + // of step as sdkstep.BuildAnswer.OutOfStep, in a failed step's result too: read before the step's state is. + var compared sdkstep.BuildAnswer if len(step.Result) > 0 && json.Unmarshal(step.Result, &compared) == nil { l.outOfStep = compared.OutOfStep } - if step.State != delivery.StepPassed { + if step.State != sdkstep.StepPassed { l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason) return } - var answer delivery.BuildAnswer + var answer sdkstep.BuildAnswer if err := json.Unmarshal(step.Result, &answer); err != nil { l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result) return } l.imports = answer.Imports - built := map[string]delivery.AskBuilt{} + built := map[string]sdkstep.AskBuilt{} for _, a := range answer.Artifacts { built[a.Artifact] = a } @@ -959,11 +963,11 @@ func (b *Builder) now() time.Time { } // logBuild tells the log an entry, said in the journal when it cannot be one. -func (b *Builder) logBuild(e delivery.BuildLogEntry) { +func (b *Builder) logBuild(e builds.BuildLogEntry) { if b.Log == nil { return } - if _, err := delivery.EncodeBuildLogEntry(e); err != nil { + if _, err := builds.EncodeBuildLogEntry(e); err != nil { b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err) return } @@ -972,7 +976,7 @@ func (b *Builder) logBuild(e delivery.BuildLogEntry) { // refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's. func (b *Builder) refOf(module, fp string) string { - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return "" } @@ -984,27 +988,27 @@ func (b *Builder) refOf(module, fp string) string { // LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the // source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder // that asked it and the time. -func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry { +func LogStarted(module string, m meshcatalog.ModulesByModuleValue, ref, by string, at time.Time) builds.BuildLogEntry { dir := m.Path if dir == "" { - dir = delivery.RootPath + dir = meshcatalogclient.RootPath } - return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10), + return builds.BuildLogEntry{Event: builds.BuildStartedEvent, Module: module, FP: m.Fp2, Source: strconv.FormatUint(uint64(m.Src), 10), Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()} } // LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it // was built against and each artifact made, as the done key has them: what the operator's command nox builds shows of a build. -func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry { +func LogEnded(module string, m meshcatalog.ModulesByModuleValue, ref, by string, at time.Time, result builds.BuilderBuild) builds.BuildLogEntry { e := LogStarted(module, m, ref, by, at) - e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest + e.Event, e.Outcome, e.Manifest = builds.BuildEndedEvent, result.State, m.Manifest if result.Manifest != "" { e.Manifest = result.Manifest } if m.Inputs2 != nil { e.Against = append([]string(nil), m.Inputs2.Bases...) } - e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...) + e.Made = append([]builds.BuiltArtifact(nil), result.Artifacts...) e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2) return e } @@ -1057,7 +1061,7 @@ func importsOf(lanes []*lane) []string { // SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version // it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2). -func SittingOutWords(language string, a delivery.Agreement) []string { +func SittingOutWords(language string, a sdkstep.Agreement) []string { var out []string for _, s := range a.SittingOut { out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version)) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 83c965e7..93081d0a 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -14,11 +14,15 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/forms" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/keys" "mesh-delivery-builder/internal/versions" ) @@ -45,7 +49,7 @@ type buildModules struct { } func newBuildModules() *buildModules { - return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}} + return &buildModules{steps: &fakeSteps{steps: map[string]buildstep.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}} } func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) { @@ -63,9 +67,9 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro id := fmt.Sprintf("s%d", b.n) module, tool, _ := strings.Cut(key, ".") args, _ := body.(map[string]any) - step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepPassed} + step := buildstep.Step{ID: id, Tool: tool, Node: node, State: buildstep.StepPassed} if reason, ok := b.fails[node+"/"+tool]; ok { - step.State, step.Reason = delivery.StepFailed, reason + step.State, step.Reason = buildstep.StepFailed, reason } else if strings.HasSuffix(tool, "_fetch") { step.Result = json.RawMessage(`{"workspace":"w-` + node + `"}`) if args["base"] != nil { // a fetch with a base answers the merge, or the files in conflict (mesh-build-go FetchResult) @@ -96,7 +100,7 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro step.Result = json.RawMessage(`{"digest":"` + d + `"}`) } } - sk, _ := delivery.StepKey(node, id) + sk, _ := buildstep.StepKey(node, id) b.steps.mu.Lock() b.steps.steps[module+"/"+sk] = step b.steps.mu.Unlock() @@ -145,12 +149,12 @@ var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun // answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the // rig's pulling node, and answers the step that node writes. -func (r *rig) answer(a delivery.StepAsk) delivery.Step { +func (r *rig) answer(a buildstep.StepAsk) buildstep.Step { node := r.puller if node == "" { node = "ace" } - module, _ := delivery.BuildModuleOf(a.Language) + module, _ := buildstep.BuildModuleOf(a.Language) runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second} tool := func(name string) stepFunc { return func(ctx context.Context, args map[string]any) (any, error) { @@ -163,9 +167,9 @@ func (r *rig) answer(a delivery.StepAsk) delivery.Step { } steps := askSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")} out, err := runAsk(context.Background(), steps, a) - step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed} + step := buildstep.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: buildstep.StepPassed} if err != nil { - step.State, step.Reason = delivery.StepFailed, err.Error() + step.State, step.Reason = buildstep.StepFailed, err.Error() return step } step.Result, _ = json.Marshal(out) @@ -186,27 +190,27 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil }, } var err error - if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil { + if r.key, err = builds.BuilderBuildKey("m", fp1); err != nil { t.Fatal(err) } return r } // agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names. -var agreedToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} +var agreedToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) } -func moduleValueWith(t *testing.T, path string, toolchains map[string]delivery.ToolchainUsed) []byte { +func moduleValueWith(t *testing.T, path string, toolchains map[string]builds.ToolchainUsed) []byte { t.Helper() - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: toolchains}) + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'), Fp2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: CatalogToolchains(toolchains)}) if err != nil { t.Fatal(err) } return data } -func (r *rig) consider(t *testing.T, path string) delivery.BuilderBuild { +func (r *rig) consider(t *testing.T, path string) builds.BuilderBuild { t.Helper() r.b.Module(context.Background(), "m", moduleValue(t, path)) r.b.Wait() @@ -237,43 +241,43 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 t.Errorf("the key does not say ace sits out: %s", st) } }() - var logged []delivery.BuildLogEntry + var logged []builds.BuildLogEntry var lmu sync.Mutex - r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() } + r.b.Node, r.b.Log = "novox", func(e builds.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() } v := r.consider(t, "modules/m") // 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read // from the two. - if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent || - logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" || + if len(logged) != 2 || logged[0].Event != builds.BuildStartedEvent || logged[1].Event != builds.BuildEndedEvent || + logged[1].Outcome != builds.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" || logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" || logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" || logged[0].FP != fp1 || logged[0].Module != "m" { t.Fatalf("logged = %+v", logged) } for _, e := range logged { - if _, err := delivery.EncodeBuildLogEntry(e); err != nil { + if _, err := builds.EncodeBuildLogEntry(e); err != nil { t.Errorf("an entry the log refuses: %v", err) } } - if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok { + if _, ok := builds.BuildDuration(logged[0], logged[1]); !ok { t.Error("no duration from the two entries") } // builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more. var page stdio.LogPage for i, e := range logged { - raw, _ := delivery.EncodeBuildLogEntry(e) + raw, _ := builds.EncodeBuildLogEntry(e) page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At}) } page.Next = 3 since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At) - if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More { + if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != builds.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More { t.Errorf("builds since = %+v, %v, want the one end and more", since, err) } page.Entries[0].Value = []byte(`{"v":0}`) if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil { t.Error("an entry that does not read was skipped, want it refused") } - if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { + if v.State != builds.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } for i, want := range []string{"files", "tool", "web"} { @@ -321,14 +325,14 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil // no node has pulled it yet // Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with. - sdk := delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')} - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", - FP2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains, Packages: sdk}) + sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')} + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", + Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk}) if err != nil { t.Fatal(err) } r.b.Module(context.Background(), "m", data) - var asks []delivery.StepAsk + var asks []buildstep.StepAsk for deadline := time.Now().Add(10 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { t.Fatal("no ask was put") @@ -336,13 +340,13 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { asks = r.asks.Asks() } a := asks[0] - if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 { + if len(asks) != 1 || a.Kind != buildstep.AskDelivery || a.Language != buildstep.AskGo || a.Build == nil || a.Build.Fp != fp1 { t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks) } if !reflect.DeepEqual(a.Packages, sdk) { t.Errorf("the ask's packages = %v, want %v", a.Packages, sdk) } - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildBuilding || v.Asks[buildstep.AskGo] != a.ID { t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID) } if n := len(r.bm.callsOf("build_go_build")) + len(r.bm.callsOf("build_go_fetch")); n != 0 { @@ -350,7 +354,7 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { } r.asks.End(r.answer(a)) // a node pulled it and ran it r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || !strings.Contains(v.Step, "on ace") || v.Artifacts[0].Reference == "" || len(v.Asks) != 0 { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || !strings.Contains(v.Step, "on ace") || v.Artifacts[0].Reference == "" || len(v.Asks) != 0 { t.Fatalf("the key = %+v, want done on ace naming the reference", v) } } @@ -360,17 +364,17 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { func TestARestartedBuilderWaitsForTheAskTheKeyNames(t *testing.T) { r := newRig(t, goBundle) const id = "mvl0frv18d1957" - data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, - Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) + data, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go", Asks: map[string]string{buildstep.AskGo: id}}) r.st.vals[r.key] = data r.asks.Answer = nil - r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, + r.asks.End(buildstep.Step{ID: id, Node: "shanks", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) v := r.consider(t, "modules/m") if n := len(r.asks.Asks()); n != 0 { t.Fatalf("%d asks put again", n) } - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on shanks") { + if v.State != builds.BuildDone || !strings.Contains(v.Step, "on shanks") { t.Fatalf("the key = %+v, want done from the ask's step on shanks", v) } } @@ -379,13 +383,13 @@ func TestAFailingStepFailsTheBuildWithItsReasonAndNodeAndIsTriedAgainAsANewRevis r := newRig(t, goBundle) r.bm.fails["ace/build_go_build"] = "go build of tool: undefined: x" v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "undefined: x") { + if v.State != builds.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "undefined: x") { t.Fatalf("build = %+v, want failed with the node and the reason", v) } rev := r.st.revs[r.key] delete(r.bm.fails, "ace/build_go_build") v = r.consider(t, "modules/m") - if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev { + if v.State != builds.BuildDone || r.st.revs[r.key] <= rev { t.Errorf("build = %+v at revision %d (was %d), want done as a new revision of the same key", v, r.st.revs[r.key], rev) } } @@ -394,7 +398,7 @@ func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { / r := newRig(t, imageArt) r.puller = "ace" // two nodes qualify; an image still builds on one v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest { t.Fatalf("build = %+v, want done with one image artifact and its digest", v) } if v.Artifacts[0].Reference == "" { @@ -414,7 +418,7 @@ func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a r := newRig(t, goBundle, imageArt) r.puller = "ace" v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || len(v.Artifacts) != 2 { + if v.State != builds.BuildDone || len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) } if n := len(r.bm.callsOf("build_go_build")); n != 1 { @@ -443,7 +447,7 @@ func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) t.Errorf("the check's build numbered with tags %v (asked %v), want feat-x for its head", nums.tags, tagAsked) } v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { t.Fatalf("build = %+v, want done with no reference", v) } calls := r.bm.callsOf("build_image_build") @@ -503,14 +507,14 @@ type fakeNumbers struct { done map[string]string // the done key's digests the last look numbered with } -func (f *fakeNumbers) Number(_ context.Context, _ string, _ delivery.Module, _ Plan, tag string, done map[string]string) (versions.Numbering, error) { +func (f *fakeNumbers) Number(_ context.Context, _ string, _ meshcatalog.ModulesByModuleValue, _ Plan, tag string, done map[string]string) (versions.Numbering, error) { f.tags = append(f.tags, tag) f.done = done f.numbered++ return versions.Numbering{Wait: f.wait, Versions: f.versions}, nil } -func (f *fakeNumbers) Built(_ context.Context, _ string, _ delivery.Module, _ Plan, got versions.Numbering, made []delivery.BuiltArtifact) error { +func (f *fakeNumbers) Built(_ context.Context, _ string, _ meshcatalog.ModulesByModuleValue, _ Plan, got versions.Numbering, made []builds.BuiltArtifact) error { f.built = map[string]string{} for _, a := range made { if got.Versions[a.Name] != "" { @@ -523,7 +527,7 @@ func (f *fakeNumbers) Built(_ context.Context, _ string, _ delivery.Module, _ Pl func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4 r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}) v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotPinned) || !strings.Contains(v.Error, "up") { + if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotPinned) || !strings.Contains(v.Error, "up") { t.Fatalf("build = %+v, want failed 'not pinned:' naming the artifact", v) } if len(r.bm.calls) != 0 { @@ -539,7 +543,7 @@ func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // func TestAModuleKeyWithNoPathFailsNamingThatReason(t *testing.T) { // 1607/AC8, ADR 0478 clause 8 r := newRig(t, goBundle) v := r.consider(t, "") - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "path") || len(r.bm.calls) != 0 { + if v.State != builds.BuildFailed || !strings.Contains(v.Error, "path") || len(r.bm.calls) != 0 { t.Errorf("build = %+v with %d calls, want failed naming the missing path and nothing called", v, len(r.bm.calls)) } } @@ -548,14 +552,14 @@ func TestAnUnreadableManifestFailsTheBuild(t *testing.T) { r := newRig(t, goBundle) delete(r.mf.byDigest, mdigest('a')) v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "manifest") { + if v.State != builds.BuildFailed || !strings.Contains(v.Error, "manifest") { t.Errorf("build = %+v, want failed naming the manifest", v) } } func TestAModuleWithNoFingerprintIsNotBuilt(t *testing.T) { // 1607/AC2 r := newRig(t) - data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"}) + data, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/r", Commit: "c", Fp: "src1:" + hex64('f'), None2: "toolchain not yet reported"}) r.b.Module(context.Background(), "n", data) r.b.Wait() if len(r.st.vals) != 0 { @@ -568,7 +572,7 @@ func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testin r := newRig(t) r.mf.byDigest[mdigest('a')] = manifest v := r.consider(t, "") // a module with nothing to build needs no directory - if v.State != delivery.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" { + if v.State != builds.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" { t.Errorf("%s: build = %+v, want done with no artifact", name, v) } if len(r.bm.calls) != 0 { @@ -585,8 +589,8 @@ func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testin func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *testing.T) { // 1607/AC3, AC7 r := newRig(t, goBundle) - seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD}) - if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone { + seed(t, r.st, builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Manifest: manifestD}) + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { t.Fatalf("build = %+v, want the builder's own building key built", v) } calls := len(r.bm.calls) @@ -611,7 +615,7 @@ func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/A go func() { defer wg.Done(); b.Module(context.Background(), "m", value); b.Wait() }() } wg.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || len(v.Artifacts) != 1 { t.Errorf("build = %+v, want one done key", v) } if n := len(r.bm.callsOf("build_go_build")); n < 1 { @@ -631,14 +635,14 @@ func TestAModulesValueThatArrivesWhileItsKeyIsBeingBuiltIsLookedAtAgainWhenItEnd delete(r.bm.fails, "ace/build_go_build") // what the event is about: the build can succeed now close(r.bm.hold) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Errorf("build = %+v, want the failed key built again as a new revision after the first build ended", v) } } func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) - old := map[string]delivery.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}} + old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}} r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old)) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { @@ -646,7 +650,7 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls)) } // A recipe that differs is a different toolchain too, and a value that names none (written before) is not built. - r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}})) + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}})) r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { @@ -660,9 +664,9 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt if len(r.st.vals) != 0 { t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals) } - r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}})) + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}})) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Toolchains["go"].Version != "go1.27.2" { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Toolchains["go"].Version != "go1.27.2" { t.Errorf("build = %+v, want done with go1.27.2", v) } } @@ -676,7 +680,7 @@ func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) { calls := len(r.bm.calls) fp2 := "src2:" + hex64('2') for _, n := range []string{"ace", "novox"} { - b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now()}) + b, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now().UTC()}) if err != nil { t.Fatal(err) } @@ -687,21 +691,21 @@ func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) { if len(r.bm.calls) != calls { t.Errorf("%d calls, was %d: the first fingerprint was built again for the new recipe", len(r.bm.calls), calls) } - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp2, - Manifest: mdigest('a'), Path: "modules/m", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}}}) + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'), Fp2: fp2, + Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}})}) if err != nil { t.Fatal(err) } r.b.Module(context.Background(), "m", data) r.b.Wait() - second, err := delivery.BuilderBuildKey("m", fp2) + second, err := builds.BuilderBuildKey("m", fp2) if err != nil { t.Fatal(err) } - if v := valueOf(t, r.st, second); v.State != delivery.BuildDone || v.Toolchains["go"].Recipe != "2" { + if v := valueOf(t, r.st, second); v.State != builds.BuildDone || v.Toolchains["go"].Recipe != "2" { t.Errorf("second build = %+v, want done with recipe 2", v) } - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Errorf("first build = %+v, want it left done", v) } } @@ -718,7 +722,7 @@ func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4 func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) - if v := r.consider(t, delivery.RootPath); v.State != delivery.BuildDone { + if v := r.consider(t, meshcatalogclient.RootPath); v.State != builds.BuildDone { t.Fatalf("build = %+v, want done for a root module", v) } if args := r.bm.callsOf("build_go_build")[0].body.(map[string]any); args["module"] != "." { @@ -733,7 +737,7 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "" { + if v.State != builds.BuildDone || v.Artifacts[0].Reference != "" { t.Fatalf("candidate build = %+v, want done with no reference", v) } if n := len(r.bm.callsOf("build_image_build")); n != 1 { @@ -743,7 +747,7 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() v = valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" { + if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" { t.Fatalf("trunk build = %+v, want done with a reference", v) } calls := r.bm.callsOf("build_image_build") @@ -771,21 +775,21 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re trunkModule(t, r) r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { t.Fatalf("candidate build = %+v", v) } r.bm.fails["ace/build_image_build"] = "push to the registry: connection refused" r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk names the fingerprint r.b.Wait() v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") { + if v.State != builds.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") { t.Fatalf("build = %+v, want failed naming the publish and its reason", v) } // The failed key is retaken as any failed key is, and ends published once the registry answers. delete(r.bm.fails, "ace/build_image_build") r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Artifacts[0].Reference == "" { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Artifacts[0].Reference == "" { t.Errorf("build = %+v, want done with a reference", v) } } @@ -795,18 +799,18 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re type goneAsks struct { mu sync.Mutex n int - why func(a delivery.StepAsk) string + why func(a buildstep.StepAsk) string dropped []string } -func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error { +func (g *goneAsks) Drop(_ context.Context, a buildstep.StepAsk) error { g.mu.Lock() defer g.mu.Unlock() g.dropped = append(g.dropped, a.ID) return nil } -func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) { +func (g *goneAsks) Gone(_ context.Context, a buildstep.StepAsk) (string, error) { g.mu.Lock() g.n++ g.mu.Unlock() @@ -820,7 +824,7 @@ func (g *goneAsks) asked() int { } // waitForAsks waits until n asks were put. -func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk { +func waitForAsks(t *testing.T, r *rig, n int) []buildstep.StepAsk { t.Helper() for deadline := time.Now().Add(10 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { @@ -838,7 +842,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} - gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }} + gone := &goneAsks{why: func(buildstep.StepAsk) string { return "" }} r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) waitForAsks(t, r, 1) @@ -849,7 +853,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { t.Fatalf("whether the ask can still be answered was read %d times within 10 s, want 5", gone.asked()) } } - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildBuilding { t.Fatalf("the key = %+v while its ask waits, want building", v) } if gone.asked() == 0 { @@ -857,7 +861,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { } r.asks.End(r.answer(r.asks.Asks()[0])) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || len(r.asks.Asks()) != 1 { t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) } }) @@ -871,17 +875,17 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { r.b.Now = func() time.Time { return start.Add(time.Duration(clock.Add(1)) * time.Second) } r.b.DeclinePause, r.b.DeclineBound = -1, time.Minute r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}}) + declined, _ := json.Marshal(buildstep.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}}) waitForAsks(t, r, 1) - for ended, deadline := 0, time.Now().Add(20*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + for ended, deadline := 0, time.Now().Add(20*time.Second); valueOf(t, r.st, r.key).State == builds.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn - r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined}) + r.asks.End(buildstep.Step{ID: asks[ended].ID, Node: "ace", State: buildstep.StepRefused, Reason: "this node builds with go1.26.0", Result: declined}) ended++ } } r.b.Wait() v := valueOf(t, r.st, r.key) - if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") { + if n := len(r.asks.Asks()); v.State != builds.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") { t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings) } }) @@ -901,44 +905,44 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { // it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it // is refused, "" when it is built again as a new revision and its ask id answered. again bool - key *delivery.BuilderBuild + key *builds.BuilderBuild known string refuse string // edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of // the error the key then holds when its build ends before its asks are put. - edit func(m *delivery.Module) + edit func(m *meshcatalog.ModulesByModuleValue) busy int failed string // during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards. during bool }{ {name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, - {name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" }, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, + {name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *meshcatalog.ModulesByModuleValue) { m.Manifest = "" }, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing", - edit: func(m *delivery.Module) { - m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}} + edit: func(m *meshcatalog.ModulesByModuleValue) { + m.Toolchains = CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}) }, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read", - edit: func(m *delivery.Module) { m.Manifest = mdigest('b') }, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, + edit: func(m *meshcatalog.ModulesByModuleValue) { m.Manifest = mdigest('b') }, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "build again: a failed key is written building as a new revision and asks the seat", again: true, - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: the step failed: no tun"}}, {name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do", - key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}}, + key: &builds.BuilderBuild{State: builds.BuildDone, Step: "built"}}, {name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting", - key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}}, + key: &builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{buildstep.AskGo: "mvl0frv18d1960"}}}, {name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry", - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: builds.NotPinned + "web"}}, {name: "build again: a key with no build is refused", again: true, refuse: "has no build key"}, {name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64), - key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, + key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2}, {name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2}, {name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2}, @@ -954,12 +958,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { c.key.Manifest = manifestD rev = putBuild(t, r.st, r.key, *c.key) } - m, err := delivery.DecodeModule(moduleValue(t, "modules/m")) + m, err := meshcatalogclient.DecodeModule(moduleValue(t, "modules/m")) if err != nil { t.Fatal(err) } if c.known != "" { - m.FP2 = c.known + m.Fp2 = c.known } if c.edit != nil { c.edit(&m) @@ -985,7 +989,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.asks.End(r.answer(waitForAsks(t, r, 1)[0])) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || res.Revision <= rev || stored.Load() != 1 { t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load()) } return @@ -1005,10 +1009,10 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { return } asks := r.asks.Asks() - if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID { + if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[buildstep.AskGo] != asks[0].ID { t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev) } - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision { + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || after <= res.Revision { t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision) } if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") { @@ -1024,8 +1028,8 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { first := "" if c.step != "" { first = "mvl0frv18d1960" - data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, - Step: c.step, Asks: map[string]string{delivery.AskGo: first}}) + data, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: c.step, Asks: map[string]string{buildstep.AskGo: first}}) r.st.vals[r.key] = data } gone := &goneAsks{} @@ -1039,7 +1043,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second} } if !c.byHand && !c.dead && !c.deaf { - gone.why = func(a delivery.StepAsk) string { + gone.why = func(a buildstep.StepAsk) string { if c.always || a.ID == first || first == "" { return "it waits first in line while a node took a later ask" } @@ -1075,7 +1079,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { first = asks[0].ID } last := asks[len(asks)-1] - if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") { + if v := valueOf(t, r.st, r.key); v.Asks[buildstep.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") { t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID) } r.asks.End(r.answer(last)) @@ -1085,10 +1089,10 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { if n := len(r.asks.Asks()); n != c.puts { t.Errorf("%d asks put, want %d", n, c.puts) } - if c.done && v.State != delivery.BuildDone { + if c.done && v.State != builds.BuildDone { t.Errorf("the key = %+v, want done from the ask asked again", v) } - if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) { + if !c.done && (v.State != builds.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) { t.Errorf("the key = %+v, want failed saying the ask could not be answered", v) } mu.Lock() @@ -1111,18 +1115,18 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { taken := func(kind string, at time.Time) *Taken { return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at} } - later := taken(delivery.AskDelivery, put.Add(time.Minute)) + later := taken(buildstep.AskDelivery, put.Add(time.Minute)) // ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none). ahead := func(n int, step string) []Queued { var qs []Queued for i := range n { at := put.Add(-time.Duration(n-i) * time.Minute) - qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step}) + qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: buildstep.AskDelivery, At: at, PutAt: at, Step: step}) } return qs } seen := func(qs []Queued, last *Taken) AskSeen { - return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last} + return AskSeen{Stream: "S", Kind: buildstep.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last} } for _, c := range []struct { name string @@ -1134,14 +1138,14 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { {"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0}, {"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0}, {"first in line, nothing taken since", seen(nil, nil), "", 0}, - {"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0}, + {"first in line, the ask taken last was asked before it", seen(nil, taken(buildstep.AskDelivery, put.Add(-time.Second))), "", 0}, {"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2}, - {"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0}, + {"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(buildstep.AskDelivery, put.Add(-time.Second))), "", 0}, // Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours. {"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0}, - {"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0}, - {"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0}, - {"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3}, + {"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, buildstep.StepFailed), later), "no longer delivers it", 0}, + {"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(buildstep.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0}, + {"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = buildstep.AskCheck; return s }(), "", 3}, } { got := c.seen.Why() if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting { @@ -1168,12 +1172,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { return nil, time.Time{}, nil }} } - key, _ := delivery.BuilderBuildKey("m", fp1) - done, _ := delivery.BuilderBuildKey("d", fp1) - building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, - Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}}) - finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"}) - running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at}) + key, _ := builds.BuilderBuildKey("m", fp1) + done, _ := builds.BuilderBuildKey("d", fp1) + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{buildstep.AskGo: "mvo53jcm70b5f7", buildstep.AskImage: "mvo53jcm70b5f8"}}) + finished, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: manifestD, Step: "built"}) + running, _ := json.Marshal(buildstep.Step{V: keys.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: buildstep.StepRunning, Started: at}) _, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at) r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}), Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})}, @@ -1192,7 +1196,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck) } a := stuck[0].Asks - if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() || + if len(a) != 2 || a[0].Language != buildstep.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() || a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") { t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a) } @@ -1203,7 +1207,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { var gets, written, inFlight, maxFlight atomic.Int64 vals := map[string]string{} for i := 0; i < 3000; i++ { - k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i)) + k, _ := builds.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i)) vals[k] = string(finished) if (i/100)%10 == 0 { vals[k] = string(building) @@ -1248,7 +1252,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { // heldAsks says an ask is held when its id is among held. type heldAsks map[string]bool -func (h heldAsks) Held(_ context.Context, a delivery.StepAsk) (bool, error) { return h[a.ID], nil } +func (h heldAsks) Held(_ context.Context, a buildstep.StepAsk) (bool, error) { return h[a.ID], nil } // 1910: a builder that stopped between naming the ask in its key and putting it left a key naming an ask neither the // stream holds nor any node ran: started again, the builder puts that ask, under the same id, once, and the key is @@ -1257,13 +1261,13 @@ func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { for _, held := range []bool{false, true} { r := newRig(t, goBundle) const id = "mvl0frv18d1958" - data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, - Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) + data, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go", Asks: map[string]string{buildstep.AskGo: id}}) r.st.vals[r.key] = data r.b.Held = heldAsks{id: held} if held { r.asks.Answer = nil - r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, + r.asks.End(buildstep.Step{ID: id, Node: "shanks", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) } v := r.consider(t, "modules/m") @@ -1274,7 +1278,7 @@ func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { case !held && (len(asks) != 1 || asks[0].ID != id): t.Errorf("the ask nobody put: asks %+v, want one under %s", asks, id) } - if v.State != delivery.BuildDone { + if v.State != builds.BuildDone { t.Errorf("held %v: the key = %+v, want done", held, v) } } @@ -1283,7 +1287,7 @@ func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { // unreadableHeld cannot tell whether an ask is held. type unreadableHeld struct{} -func (unreadableHeld) Held(context.Context, delivery.StepAsk) (bool, error) { +func (unreadableHeld) Held(context.Context, buildstep.StepAsk) (bool, error) { return false, errors.New("permissions violation for publish to $JS.API.STREAM.INFO.SEAT_NODE_BUILD_GO") } @@ -1292,15 +1296,15 @@ func (unreadableHeld) Held(context.Context, delivery.StepAsk) (bool, error) { func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) { r := newRig(t, goBundle) const id = "mvl0frv18d1959" - data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, - Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) + data, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go", Asks: map[string]string{buildstep.AskGo: id}}) r.st.vals[r.key] = data r.b.Held = unreadableHeld{} r.asks.Answer = nil - r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, + r.asks.End(buildstep.Step{ID: id, Node: "shanks", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || len(r.asks.Asks()) != 0 { + if v.State != builds.BuildDone || len(r.asks.Asks()) != 0 { t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks())) } } @@ -1308,8 +1312,8 @@ func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) { // silentWatch is a watch of an ask's step that stays open and hears nothing. type silentWatch struct{} -func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) { - return make(chan delivery.Step), nil +func (silentWatch) WatchAsk(context.Context, string, string) (<-chan buildstep.Step, error) { + return make(chan buildstep.Step), nil } // aDoneKeyKeepsEachFormACountedNodeReads holds the builder's rows of ADR 0624's check: every build keeps each form a @@ -1344,7 +1348,7 @@ func aDoneKeyKeepsEachFormACountedNodeReads(t *testing.T) { if got := fs.Kept("bundle", row.present, row.counted); !slices.Equal(got, row.kept) { t.Errorf("%s: kept %v, want %v", row.name, got, row.kept) } - a := fs.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{newRef, oldRef}, row.present, row.counted) + a := fs.Choose(forms.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{newRef, oldRef}, row.present, row.counted) if a.Reference != row.reference || !slices.Equal(a.References, row.references) { t.Errorf("%s: %q %v, want %q %v", row.name, a.Reference, a.References, row.reference, row.references) } @@ -1354,11 +1358,11 @@ func aDoneKeyKeepsEachFormACountedNodeReads(t *testing.T) { } } // An answered reference no kept form names still names where the artifact is (ADR 0498 clause 1). - if a := fs.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{oldRef}, map[string][]string{"a": both}, + if a := fs.Choose(forms.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{oldRef}, map[string][]string{"a": both}, map[string][]string{"a": both}); a.Reference != oldRef { t.Errorf("a lone answered reference was dropped: %+v", a) } - withBoth := []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: digest, Reference: oldRef, References: []string{oldRef, newRef}}} + withBoth := []forms.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: digest, Reference: oldRef, References: []string{oldRef, newRef}}} if got := fs.Needs(withBoth); !slices.Equal(got, []string{"form:new", "form:old"}) { t.Errorf("needs %v", got) } @@ -1390,11 +1394,11 @@ func aDoneKeyKeepsEachFormACountedNodeReads(t *testing.T) { map[string][]string{"a": seat, "b": nil}, map[string][]string{"a": seat, "b": nil}, storeRef, nil, []string{"form:artifact-store"}}, {"seat form answered, no node present: the store form", nil, map[string][]string{"a": seat}, storeRef, nil, []string{"form:artifact-store"}}, } { - a := forms.Current.Choose(delivery.BuiltArtifact{Name: "code", Kind: k.kind, Digest: digest}, []string{seatRef}, row.present, row.counted) + a := forms.Current.Choose(forms.BuiltArtifact{Name: "code", Kind: k.kind, Digest: digest}, []string{seatRef}, row.present, row.counted) if a.Reference != row.reference || !slices.Equal(a.References, row.references) { t.Errorf("%s %s: %q %v, want %q %v", k.kind, row.name, a.Reference, a.References, row.reference, row.references) } - needs := forms.Current.Needs([]delivery.BuiltArtifact{a}) + needs := forms.Current.Needs([]forms.BuiltArtifact{a}) if !slices.Equal(needs, row.needs) { t.Errorf("%s %s: needs %v, want %v", k.kind, row.name, needs, row.needs) } @@ -1403,7 +1407,7 @@ func aDoneKeyKeepsEachFormACountedNodeReads(t *testing.T) { t.Errorf("%s %s: the want of %s is held", k.kind, row.name, node) } } - if _, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.TrimPrefix(digest, "sha256:"), Artifacts: []delivery.BuiltArtifact{a}, Needs: needs}); err != nil { + if _, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.TrimPrefix(digest, "sha256:"), Artifacts: []builds.BuiltArtifact{a}, Needs: needs}); err != nil { t.Errorf("%s %s: %v", k.kind, row.name, err) } } diff --git a/modules/mesh-delivery-builder/internal/build/since.go b/modules/mesh-delivery-builder/internal/build/since.go index 968e5979..8875b37d 100644 --- a/modules/mesh-delivery-builder/internal/build/since.go +++ b/modules/mesh-delivery-builder/internal/build/since.go @@ -4,8 +4,8 @@ import ( "fmt" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // BuildsSinceLimit is how many entries one answer of BuildsSinceTool reads of the log; More says the log held more. @@ -14,28 +14,28 @@ const BuildsSinceLimit = 500 // LogSince is one page of the builder's log from a time on: stdio.Log(BuilderLog).Since. type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) -// BuildsSince answers delivery.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on, +// BuildsSince answers builds.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on, // oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again // from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that // skipped it would register too little. -func BuildsSince(read LogSince, since time.Time) (delivery.BuildsSince, error) { - out := delivery.BuildsSince{Since: since.UTC(), Builds: []delivery.BuildLogEntry{}} +func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) { + out := builds.BuildsSince{Since: since.UTC(), Builds: []builds.BuildLogEntry{}} page, err := read(since, "", BuildsSinceLimit) if err != nil { - return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuilderLog, since.Format(time.RFC3339), err) + return out, fmt.Errorf("the builder's log %s from %s: %w", builds.BuilderLog, since.Format(time.RFC3339), err) } if page == nil { return out, nil } - var entries []delivery.BuildLogEntry + var entries []builds.BuildLogEntry for _, le := range page.Entries { - e, err := delivery.DecodeBuildLogEntry(le.Value) + e, err := builds.DecodeBuildLogEntry(le.Value) if err != nil { - return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuilderLog, le.Sequence, err) + return out, fmt.Errorf("the builder's log %s, entry %d: %w", builds.BuilderLog, le.Sequence, err) } entries = append(entries, e) } - if ended := delivery.EndedSince(entries, since); ended != nil { + if ended := builds.EndedSince(entries, since); ended != nil { out.Builds = ended } out.More = page.Next != 0 diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 93dd5703..e5ef2e86 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -14,7 +14,8 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) // The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads @@ -31,7 +32,7 @@ const ( AnnounceAgainTool = "builder_announce_again" // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. ReAsksState = "reasks" - ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState + ReAsksBucket = builds.BuilderModule + "_" + ReAsksState ) // ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder @@ -76,7 +77,7 @@ func NewPublishAgain(name, fp, why, by string, at time.Time) (string, []byte, er if !ok || module == "" || artifact == "" { return "", nil, fmt.Errorf("name %q is not a numbered name, /, as builder_versions lists it", name) } - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return "", nil, err } @@ -139,7 +140,7 @@ func DecodeReAsk(data []byte) (ReAsk, error) { // NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value. func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) { - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return "", nil, err } @@ -152,7 +153,7 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) // NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value. func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) { - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return "", nil, err } @@ -165,7 +166,7 @@ func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, er // NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value. func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) { - key, err := delivery.BuilderBuildKey(module, fp) + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return "", nil, err } @@ -303,7 +304,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { } byID := make(map[string]string, len(ks)) for _, k := range ks { - if _, id, err := delivery.ParseStepKey(k); err == nil { + if _, id, err := buildstep.ParseStepKey(k); err == nil { byID[id] = k } } @@ -314,7 +315,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { type parsed struct{ key, mod, fp string } var matching []parsed for _, key := range keys { - mod, fp, err := delivery.ParseBuilderBuildKey(key) + mod, fp, err := builds.ParseBuilderBuildKey(key) if err != nil || (module != "" && mod != module) { continue } @@ -345,8 +346,8 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { if raw == nil { continue } - v, err := delivery.DecodeBuilderBuild(raw) - if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) { + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || (v.State != builds.BuildBuilding && v.State != builds.BuildQueued) { continue } if at.IsZero() && r.Builds.Written != nil { @@ -357,7 +358,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { s.Age = now().Sub(at).Round(time.Second).String() } for language, id := range v.Asks { - bm, _ := delivery.BuildModuleOf(language) + bm, _ := buildstep.BuildModuleOf(language) a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"} if t, ok := idTime(id); ok { a.Asked = t @@ -366,7 +367,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { if k, ok := stepKeys[bm][id]; ok { reads++ if raw, _, err := st.Get(k); err == nil && raw != nil { - if step, err := delivery.DecodeStep(raw); err == nil { + if step, err := buildstep.DecodeStep(raw); err == nil { a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339)) } } @@ -378,7 +379,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { s.Asks = append(s.Asks, a) } sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language }) - s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0 + s.Unasked = v.State == builds.BuildBuilding && len(v.Asks) == 0 if r.ReAsks.Get != nil { reads++ if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil { diff --git a/modules/mesh-delivery-builder/internal/build/toolchains.go b/modules/mesh-delivery-builder/internal/build/toolchains.go new file mode 100644 index 00000000..f2477c0a --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/toolchains.go @@ -0,0 +1,28 @@ +package build + +import ( + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// CatalogToolchains are the toolchains of a build, by language, as a module's value holds them (the catalog's form names +// the language in the entry). +func CatalogToolchains(used map[string]builds.ToolchainUsed) map[string]meshcatalog.ToolchainUsed { + if used == nil { + return nil + } + out := make(map[string]meshcatalog.ToolchainUsed, len(used)) + for language, t := range used { + out[language] = meshcatalog.ToolchainUsed{Language: language, Recipe: t.Recipe, Version: t.Version} + } + return out +} + +// buildToolchain is the toolchain a build ask names for a language (the ask's form names the language in the entry). +func buildToolchain(language string, t *builds.ToolchainUsed) *sdkstep.ToolchainUsed { + if t == nil { + return nil + } + return &sdkstep.ToolchainUsed{Language: language, Recipe: t.Recipe, Version: t.Version} +} diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 8ef89b13..9e77235b 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // What builder_publish_again does with one fingerprint of a name (design 51 *Tools of the builder*), after a holder failed. @@ -309,7 +309,7 @@ func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at tim b.Again = at.UTC().Format(time.RFC3339Nano) a.Numberer.Said(b) } - answer += "; " + delivery.VersionNumberedEvent + " said for " + ag.Entry.Version + answer += "; " + builds.VersionNumberedEvent + " said for " + ag.Entry.Version } return ag, answer, nil } diff --git a/modules/mesh-delivery-builder/internal/versions/claim.go b/modules/mesh-delivery-builder/internal/versions/claim.go index a9990b02..16d3e2a9 100644 --- a/modules/mesh-delivery-builder/internal/versions/claim.go +++ b/modules/mesh-delivery-builder/internal/versions/claim.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // The registry seats a version is published to (design 51 *The seats*), by what is published. @@ -139,7 +139,7 @@ type Numberer struct { // Holder is the module holding a seat, whose health condition's clear wakes a silent seat source; nil names none. Holder func(seat string) string // Said is told each version newly numbered; nil tells nobody. - Said func(delivery.VersionNumbered) + Said func(builds.VersionNumbered) // Names answers a package's name in its ecosystem; nil numbers a package as NameOf names it. Names PackageNames // Publishing is told each version Built recorded that is still numbered, its holder's word not heard yet: the diff --git a/modules/mesh-delivery-builder/internal/versions/names.go b/modules/mesh-delivery-builder/internal/versions/names.go index 0d01172e..66dd54d0 100644 --- a/modules/mesh-delivery-builder/internal/versions/names.go +++ b/modules/mesh-delivery-builder/internal/versions/names.go @@ -5,7 +5,7 @@ import ( "sort" "strings" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" ) // PackagesState is the indexer's packages state as the builder reads it (its manifest's reads grant): its keys, and one @@ -27,7 +27,7 @@ func NamesIn(s PackagesState) PackageNames { sort.Strings(keys) built := NameOf(module, artifact) for _, key := range keys { - if !delivery.IsSeriesPackageKey(key) { + if !isSeriesKey(key) { continue } value, found, err := s.Get(ctx, key) @@ -37,7 +37,7 @@ func NamesIn(s PackagesState) PackageNames { if !found { continue } - v, err := delivery.DecodeSeriesPackage(value) // the indexer's value of a series key (mesh-sdk's contract, J4) + v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value) // the indexer's value of a series key (mesh-sdk's contract, J4) if err != nil || v.Package != built { continue } @@ -51,10 +51,10 @@ func NamesIn(s PackagesState) PackageNames { // name: a series key whose value names a name and the / it is built from. The builder looks at that module again when // the value is heard, so a done build that left the package unnumbered (its name not known at its claim) is numbered then. func NamedModule(key string, value []byte) (module, name string, ok bool) { - if !delivery.IsSeriesPackageKey(key) { + if !isSeriesKey(key) { return "", "", false } - v, err := delivery.DecodeSeriesPackage(value) + v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value) if err != nil || v.Name == "" { return "", "", false } @@ -64,3 +64,9 @@ func NamedModule(key string, value []byte) (module, name string, ok bool) { } return module, v.Name, true } + +// isSeriesKey says whether a key of the indexer's packages state is a series key (`.`), not a per-module one. +func isSeriesKey(key string) bool { + _, _, ok := meshcatalog.ParsePackagesByPackageSeriesKey(key) + return ok +} diff --git a/modules/mesh-delivery-builder/internal/versions/oldform.go b/modules/mesh-delivery-builder/internal/versions/oldform.go index e2dba2ea..dd066d94 100644 --- a/modules/mesh-delivery-builder/internal/versions/oldform.go +++ b/modules/mesh-delivery-builder/internal/versions/oldform.go @@ -9,8 +9,9 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/forms" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/sdk/window" ) // The old reference form's window (design 51 *Every build keeps the old reference form*, ADR 0624 §1, ADR 0618 §2-§4, @@ -28,7 +29,7 @@ const ( ) // OldFormBucket is that state's bucket. -const OldFormBucket = delivery.BuilderModule + "_" + OldFormState +const OldFormBucket = builds.BuilderModule + "_" + OldFormState // OldFormHeld is the finding of a node holding the window open for FindingAfter, about that node, cleared when the // window closes or the node no longer holds it. @@ -64,11 +65,11 @@ func Holding(fs forms.ArtifactForms, counted map[string][]string, dropped map[st return out } -// OldFormWindow is the window's record: whether it is open, who holds it and since when (delivery.WindowHeld, the part +// OldFormWindow is the window's record: whether it is open, who holds it and since when (window.WindowHeld, the part // every window keeper shares), and the nodes dropped by hand; what builder_old_form answers and what `old-form-closed` // carries when it closes. type OldFormWindow struct { - delivery.WindowHeld + window.WindowHeld Dropped []Dropped `json:"dropped,omitempty"` } @@ -91,13 +92,13 @@ type OldFormClosedBody struct { // FindingAfter is how long a node holds the window open before it is the builder's finding naming the node. const FindingAfter = 7 * 24 * time.Hour -// Step moves the window by mesh-sdk's one window rule (delivery.StepWindow): named is every counted node the read named, +// Step moves the window by mesh-sdk's one window rule (window.StepWindow): named is every counted node the read named, // those dropped by hand included, and holding those of them holding the window (Holding). A read naming no node (no // placement read yet, an empty node_haves) judges nothing, so it never closes the window. It answers the new window, // whether it closed in this step (so the builder says `old-form-closed` and its log line once), and the nodes holding it // past FindingAfter (the finding, cleared for every other node). func (w OldFormWindow) Step(named, holding []string, now time.Time) (next OldFormWindow, closed bool, overdue []string) { - held, closed, overdue := delivery.StepWindow(w.WindowHeld, named, holding, now, FindingAfter) + held, closed, overdue := window.StepWindow(w.WindowHeld, named, holding, now, FindingAfter) return OldFormWindow{WindowHeld: held, Dropped: w.Dropped}, closed, overdue } diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index 726cf915..bb8d94f0 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -7,7 +7,7 @@ import ( "fmt" "strings" - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // The builder's tools of its versions (design 51 *Events of the builder*, *Tools of the builder*). @@ -19,14 +19,14 @@ const ( TagAsksTool = "builder_tag_asks" UntagTool = "builder_untag" OldFormTool = "builder_old_form" - // OldFormClosedEvent is, with delivery.VersionNumberedEvent, the builder's own family `version`, kept seven days, + // OldFormClosedEvent is, with builds.VersionNumberedEvent, the builder's own family `version`, kept seven days, // apart from its builds' ends, so a waiter on a build's end does not wake on them. OldFormClosedEvent = "version.old-form-closed" ) -// NumberedOf is the version.numbered body of an entry (delivery.VersionNumbered, the contract a build module reads). -func NumberedOf(e Entry, head string) delivery.VersionNumbered { - return delivery.VersionNumbered{Name: e.Name, Kind: e.Kind, Version: e.Version, Tag: e.Tag, Module: e.Module, Fingerprint: e.Fingerprint, +// NumberedOf is the version.numbered body of an entry (builds.VersionNumbered, the contract a build module reads). +func NumberedOf(e Entry, head string) builds.VersionNumbered { + return builds.VersionNumbered{Name: e.Name, Kind: e.Kind, Version: e.Version, Tag: e.Tag, Module: e.Module, Fingerprint: e.Fingerprint, Commit: e.Commit, Digest: e.Digest, Seat: e.Seat, Head: head} } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index d875586c..da2fbda5 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/forms" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // mem is a Store in memory; before runs once before the next Create, to stage a race. @@ -364,7 +364,7 @@ func TestTheBuildersVersions(t *testing.T) { if _, err := w.Drop("e", "op@shanks", "", t0); err == nil { t.Error("a drop with no why is taken") } - // Security-upgrade F3 (review round 1), by mesh-sdk's delivery.StepWindow: a read naming no node (an empty + // Security-upgrade F3 (review round 1), by mesh-sdk's window.StepWindow: a read naming no node (an empty // node_haves, the placement not read yet) never closes the window, and its holder's seven days still count. if held, shut, late := w.Step(nil, nil, t0.Add(FindingAfter+time.Minute)); shut || !held.Open || !reflect.DeepEqual(late, []string{"c"}) { t.Fatalf("a read naming no node: %+v closed=%v overdue %v, want the window open and c overdue", held, shut, late) @@ -396,9 +396,9 @@ func TestTheBuildersVersions(t *testing.T) { }, Requirements: func(context.Context, string) ([]string, error) { return nil, nil }, } - var said []delivery.VersionNumbered + var said []builds.VersionNumbered fw := &FloorWaits{} - n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Waits: fw, Said: func(b delivery.VersionNumbered) { said = append(said, b) }} + n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Waits: fw, Said: func(b builds.VersionNumbered) { said = append(said, b) }} c := BuildClaim{Module: "m", Path: "go", Repository: "novox/m", Fingerprint: "fp1", Commit: "c1", Revision: "7", Merge: 7, Artifacts: []ClaimArtifact{{Artifact: "program", Kind: KindBundle, Language: "go"}, {Artifact: "base", Kind: "upstream"}}} got, err := n.Number(ctx, c) @@ -495,8 +495,8 @@ func TestTheBuildersVersions(t *testing.T) { return nil, nil }, } - var said []delivery.VersionNumbered - n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Names: NamesIn(state), Said: func(b delivery.VersionNumbered) { said = append(said, b) }} + var said []builds.VersionNumbered + n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Names: NamesIn(state), Said: func(b builds.VersionNumbered) { said = append(said, b) }} c := BuildClaim{Module: "mesh-sdk", Path: "go", Repository: "novox/mesh-sdk", Fingerprint: "fp1", Commit: "c1", Revision: "3", Merge: 3, Artifacts: []ClaimArtifact{{Artifact: "go", Kind: KindPackage, Language: "go"}, {Artifact: "ts", Kind: KindPackage, Language: "typescript"}, {Artifact: "new", Kind: KindPackage, Language: "go"}, {Artifact: "program", Kind: KindBundle, Language: "go"}}} @@ -642,10 +642,10 @@ func TestTheBuildersVersions(t *testing.T) { t.Error("a version refused for other bytes is published again") } _, _ = v.Mark(ctx, "m/p", "0.1.4", func(e *Entry) { e.State = Published }) - var said []delivery.VersionNumbered + var said []builds.VersionNumbered var rebuilt []string done := true - acts := Acts{Numberer: &Numberer{Versions: v, Said: func(b delivery.VersionNumbered) { said = append(said, b) }, Waits: &FloorWaits{}}, + acts := Acts{Numberer: &Numberer{Versions: v, Said: func(b builds.VersionNumbered) { said = append(said, b) }, Waits: &FloorWaits{}}, Done: func(context.Context, string, string) (bool, error) { return done, nil }, BuildAgain: func(_ context.Context, m, fp, _, _ string) (string, bool) { rebuilt = append(rebuilt, m+" "+fp)