From 689947bec5995607107219ec3648b45874e61141 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:47:42 +0100 Subject: [PATCH 1/9] BL11: a build whose published packages cannot be read is refused, naming the unread state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taken from f051de32 (the old numbering's refusal); the tests of D-B2 to D-B6 it sat on are left out. Builds: design 47 G §55 (and its check line §56, test 70) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/build/numbers.go | 4 ++++ .../mesh-delivery-builder/internal/build/run.go | 6 +++++- .../internal/build/run_test.go | 13 ++++++++++++- .../internal/versions/claim.go | 9 +++++++++ .../internal/versions/versions_test.go | 15 +++++++++++++++ 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 2fc70614..f8d7ee45 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -99,6 +99,10 @@ func (b *Builder) number(ctx context.Context, module string, m meshcatalog.Modul b.logf("%s at %s: %v", module, m.Fp2, fmt.Errorf("not numbered, the versions cannot be used: %w", err)) return versions.Numbering{}, true } + if got.Refused != "" { + b.logf("%s at %s: %s", module, m.Fp2, got.Refused) + return got, true + } if got.Wait != "" { b.logf("%s at %s is not built yet: %s", module, m.Fp2, got.Wait) return got, false diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 4a242237..baaf480c 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -539,7 +539,11 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB reserved, hasReserved = b.reserve(ctx, module, m, plan) } var result builds.BuilderBuild - if nothing { + if numbering.Refused != "" && cl.Build.State != builds.BuildDone { + // Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key + // fails with why, and is taken again at the next look. + result = failed("%s", numbering.Refused) + } else if nothing { // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). result = builds.BuilderBuild{State: builds.BuildDone, Step: "nothing to build: the manifest names no artifact"} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index cfc3ad9b..cfe21eaa 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -495,6 +495,16 @@ func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The if nums.numbered != looks+1 { t.Errorf("a module the builder does not know was looked at") } + // D-B5 (ADR 0666 §15; design 47, test 70): a build whose published packages cannot be read is refused, naming the + // state: its key fails with why, and nothing is built. + unread := newRig(t, imageArt) + unread.puller = "ace" + unread.b.Numbers = &fakeNumbers{refused: "m/img: refused: the builder's state versions (bucket mesh-delivery-builder_versions) cannot be read"} + unread.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + unread.b.Wait() + if v := valueOf(t, unread.st, unread.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "state versions") || len(unread.bm.callsOf("build_image_build")) != 0 { + t.Errorf("a build with its versions unread: key %+v, %d builds; want failed naming the state, none built", v, len(unread.bm.callsOf("build_image_build"))) + } } // fakeNumbers is Numbers that answers what it is set to and keeps the digests a done build recorded. @@ -505,13 +515,14 @@ type fakeNumbers struct { built map[string]string tags []string done map[string]string // the done key's digests the last look numbered with + refused string // answered as Numbering.Refused } func (f *fakeNumbers) Number(_ context.Context, _ string, _ meshcatalog.ModulesByModuleValue, _ Plan, tag string, done map[string]string) (versions.Numbering, error) { f.tags = append(f.tags, tag) f.done = done f.numbered++ - return versions.Numbering{Wait: f.wait, Versions: f.versions}, nil + return versions.Numbering{Wait: f.wait, Versions: f.versions, Refused: f.refused}, nil } func (f *fakeNumbers) Built(_ context.Context, _ string, _ meshcatalog.ModulesByModuleValue, _ Plan, got versions.Numbering, made []builds.BuiltArtifact) error { diff --git a/modules/mesh-delivery-builder/internal/versions/claim.go b/modules/mesh-delivery-builder/internal/versions/claim.go index 16d3e2a9..912b7a2c 100644 --- a/modules/mesh-delivery-builder/internal/versions/claim.go +++ b/modules/mesh-delivery-builder/internal/versions/claim.go @@ -91,6 +91,9 @@ type Numbering struct { // Names are the names each numbered artifact was numbered under, by artifact (NameOf, or a package's name in its // ecosystem). Names map[string]string + // Refused is why the build is refused: a package's published versions could not be read (ADR 0666 §15). Nothing else + // is numbered. + Refused string } // NameOf is the name artifact of module was numbered under in this numbering. @@ -177,6 +180,12 @@ func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) } out.Names[a.Artifact] = name entries, err := n.Versions.Of(ctx, name) + if err != nil && kind == KindPackage { + // A build that cannot read the published packages refuses, naming the unread state, never building against a + // guess (ADR 0666 §15; design 51 *The versions*, line 287). + out.Refused = fmt.Sprintf("%s: refused: the builder's state %s (bucket %s) cannot be read, so its published versions are not known: %v", name, State, Bucket, err) + return out, nil + } if err != nil { return out, err } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index d94a2fa6..9bb80e17 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -23,6 +23,7 @@ type mem struct { revs map[string]uint64 rev uint64 before func() + down error // answered by KeysOf: a state that cannot be read } func newMem() *mem { return &mem{vals: map[string][]byte{}, revs: map[string]uint64{}} } @@ -63,6 +64,9 @@ func (m *mem) Update(_ context.Context, key string, value []byte, revision uint6 func (m *mem) KeysOf(_ context.Context, filter string) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() + if m.down != nil { + return nil, m.down + } prefix := strings.TrimSuffix(filter, "*") var out []string for k := range m.vals { @@ -472,6 +476,17 @@ func TestTheBuildersVersions(t *testing.T) { if _, err := n.Heard(ctx, SeatArtifacts, HeardPublished, "e4", Outcome{Name: "m/none", Version: "0.1.0"}); err == nil { t.Fatal("a published of a version the builder never numbered is not said") } + // D-B5 (ADR 0666 §15, design 51 *The versions* line 287): a package's build whose published versions cannot be read + // is refused, naming the state; a bundle's is built as before (ADR 0617 §1), the error said. + st := n.Versions.Store.(*mem) + st.down = errors.New("nats: no responders available for request") + if got, err := n.Number(ctx, pkg); err != nil || !strings.Contains(got.Refused, "state versions (bucket mesh-delivery-builder_versions) cannot be read") || len(got.Versions) != 0 { + t.Errorf("a package with its versions unread = %+v, %v; want refused naming the state", got, err) + } + if got, err := n.Number(ctx, c); err == nil || got.Refused != "" { + t.Errorf("a bundle with its versions unread = %+v, %v; want the error, no refusal", got, err) + } + st.down = nil }) t.Run("a package is numbered under its name in its ecosystem", func(t *testing.T) { // design 32: the indexer's packages state names, in each series key, the package's name and the / From 2c8d3f5b403f99ff25afb8c2d91acecc144567f7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:48:04 +0100 Subject: [PATCH 2/9] BL5 (version half): a build whose catalog entry's version is not MAJOR.MINOR reserves nothing and says so in the journal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first-watched-branch half (§371) waits for BL6's revision view, which names a build's branch. Builds: design 47 G §505 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-builder/internal/build/numbers.go | 2 ++ .../mesh-delivery-builder/internal/build/reserve_test.go | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index f8d7ee45..279a8345 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -165,6 +165,8 @@ func (b *Builder) reserves(ctx context.Context, module string, m meshcatalog.Mod return false } if strings.Count(plan.Series, ".") != 1 { + // G §505: such an entry (a manifest from before G, "1") reserves nothing and is built as a build without a version. + b.logf("%s: the catalog entry's version %q is not MAJOR.MINOR: no version is reserved, and the build is made and delivered without one", module, plan.Series) return false } return b.doneDigests(ctx, module, m.Fp2) == nil diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index b9a33735..d2c85fdf 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -2,6 +2,7 @@ package build import ( "context" + "fmt" "slices" "strings" "testing" @@ -80,9 +81,15 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { r2 := newRig(t, goBundle) res2 := &fakeReservations{filled: true} r2.b.Reserved = res2 + var journal []string + r2.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) } if v := r2.consider(t, "modules/m"); v.State != builds.BuildDone || len(res2.reserved) != 0 { t.Errorf("a manifest with no MAJOR.MINOR: state %s, reserved %v", v.State, res2.reserved) } + // G §505: the builder writes in its journal, naming the module, that it reserved nothing. + if !slices.ContainsFunc(journal, func(l string) bool { return strings.HasPrefix(l, "m: ") && strings.Contains(l, "not MAJOR.MINOR") }) { + t.Errorf("journal %q, want a line naming m and saying its version is not MAJOR.MINOR", journal) + } } // TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through From 996b15cb45a9ec8431d080f5f7dc54ae1e4bc39c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:26:55 +0100 Subject: [PATCH 3/9] BL13: build.ended names the node that keeps a kept build in kept-on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A build the build node kept, which answers its digest and no reference, is done; its end carries kept-on once for the build. Builds: design 47 G §671, §379, §271, §526 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-builder/builds/builds.go | 15 +++++++--- modules/mesh-delivery-builder/builds/log.go | 3 ++ .../internal/build/imports_test.go | 28 +++++++++++++++++++ .../internal/build/run.go | 10 ++++++- .../internal/build/run_test.go | 1 + 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index 19299958..f282bb74 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -12,7 +12,7 @@ import ( "strings" "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey" "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms" @@ -63,13 +63,13 @@ const ( ) // BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `.`, the hex -// being the fingerprint's key part (meshcatalogclient.KeyPart), since a bus key may not hold ':'. A module holding a dot or a +// being the fingerprint's key part (meshcatalogkey.KeyPart), since a bus key may not hold ':'. A module holding a dot or a // character a key refuses, and a fingerprint not of the new form, are refused, naming them. func BuilderBuildKey(module, fp string) (string, error) { if err := keys.CheckName("module", module); err != nil { return "", err } - part, err := meshcatalogclient.KeyPart(fp) + part, err := meshcatalogkey.KeyPart(fp) if err != nil { return "", err } @@ -85,7 +85,7 @@ func ParseBuilderBuildKey(key string) (module, fp string, err error) { if err := keys.CheckName("module", module); err != nil { return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err) } - if fp, err = meshcatalogclient.ParseKeyPart(part); err != nil { + if fp, err = meshcatalogkey.ParseKeyPart(part); err != nil { return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err) } return module, fp, nil @@ -237,9 +237,16 @@ type BuilderBuild struct { // node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field // reads the build as needing none. Needs []string `json:"needs,omitempty"` + // KeptOn is, on a done key only, the build node that keeps the build instead of storing it through the seat + // oci-artifact-registry (design 47 G §671): every artifact of the build is kept there, each with its digest in + // Artifacts. A build not kept names none; the builder carries it onto the build's end (BuildLogEntry.KeptOn). + KeptOn string `json:"kept-on,omitempty"` } func (b BuilderBuild) check() error { + if b.KeptOn != "" && b.State != BuildDone { + return fmt.Errorf("builds: a build is kept on a done key alone, this one is %s", b.State) + } if err := trunk.CheckImports(b.Imports); err != nil { return err } diff --git a/modules/mesh-delivery-builder/builds/log.go b/modules/mesh-delivery-builder/builds/log.go index 0c59e696..455000d2 100644 --- a/modules/mesh-delivery-builder/builds/log.go +++ b/modules/mesh-delivery-builder/builds/log.go @@ -67,6 +67,9 @@ type BuildLogEntry struct { Manifest string `json:"manifest,omitempty"` Against []string `json:"against,omitempty"` Made []BuiltArtifact `json:"made,omitempty"` + // KeptOn is, at the end of a kept build, the build node that keeps it, once for the build (design 47 G §671): every + // artifact in Made is kept there, with its digest. A build not kept carries none. + KeptOn string `json:"kept-on,omitempty"` // Imports are, at a build's end, the mesh packages (import paths) the built programs import, their tests left out, // sorted and each once, as the build module answered them (BuildAnswer.Imports; novox/hq ADR 0593 §2): the indexer // keeps them on the module (KeptInputs2.Imports) when the build's fingerprint is the module's. diff --git a/modules/mesh-delivery-builder/internal/build/imports_test.go b/modules/mesh-delivery-builder/internal/build/imports_test.go index 20ee96e5..ac12fc0f 100644 --- a/modules/mesh-delivery-builder/internal/build/imports_test.go +++ b/modules/mesh-delivery-builder/internal/build/imports_test.go @@ -82,3 +82,31 @@ func outOfStepFilesABuildAnswersReachTheLog(t *testing.T) { }) } } + +// Design 47 G §671: a build the build node kept instead of storing it, which answers its digest and no reference, is done, +// and its end in the log names the node that keeps it once; a build not kept names none. +func aKeptBuildNamesItsNodeInItsEnd(t *testing.T) { + r := newRig(t, goBundle) + r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { + step := r.answer(a) + var answer buildstep.BuildAnswer + if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" && len(answer.Artifacts) > 0 { + for i := range answer.Artifacts { + answer.Artifacts[i].Reference = "" + } + answer.Kept = strings.TrimPrefix(answer.Artifacts[0].Digest, "sha256:") + step.Result, _ = json.Marshal(answer) + } + return step + } + var logged []builds.BuildLogEntry + var mu sync.Mutex + r.b.Node, r.b.Log = "novox", func(e builds.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() } + v := r.consider(t, "modules/m") + if v.State != builds.BuildDone || v.KeptOn == "" { + t.Fatalf("done key %+v, want it done and kept on its node", v) + } + if len(logged) != 2 || logged[1].KeptOn != v.KeptOn || logged[0].KeptOn != "" { + t.Fatalf("log %+v, want kept-on %q on the end alone", logged, v.KeptOn) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index baaf480c..e17baad3 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -651,6 +651,7 @@ type lane struct { digests map[string]string // artifact -> digest references map[string]string // artifact -> the reference an image was pushed to (a trunk build only) imports []string // the mesh packages the lane's programs import, as its build module answered them + kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty outOfStep []string // the generated files its build module's compare found out of step, passed or failed err error } @@ -763,7 +764,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module continue } r := l.references[t.Artifact] - if r == "" && stored(t.Kind) { + if r == "" && stored(t.Kind) && (l.kept == "" || strings.TrimPrefix(d, "sha256:") != l.kept) { // A bundle, an archive or an upstream copy is done only when its push was answered with where it is (ADR 0498 // clause 1); a candidate's included (clause 3). return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", builds.NotStored, t.Artifact, l.node) @@ -796,6 +797,11 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module } done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) + for _, l := range lanes { + if l.kept != "" { + done.KeptOn = l.node // design 47 G §671: the build node that keeps the build + } + } if _, err := builds.EncodeBuilderBuild(done); err != nil { // A reference the build module answered that does not name the artifact and its digest is no reference. return failed("%s%v", builds.NotStored, err) @@ -1008,6 +1014,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes return } l.imports = answer.Imports + l.kept = answer.Kept built := map[string]sdkstep.AskBuilt{} for _, a := range answer.Artifacts { built[a.Artifact] = a @@ -1092,6 +1099,7 @@ func LogEnded(module string, m meshcatalog.ModulesByModuleValue, ref, by string, e.Against = append([]string(nil), m.Inputs2.Bases...) } e.Made = append([]builds.BuiltArtifact(nil), result.Artifacts...) + e.KeptOn = result.KeptOn e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2) e.Error, e.Step = result.Error, result.Step // a failed build's reason, not only its outcome return e diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index cfe21eaa..0339f41e 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -230,6 +230,7 @@ var ( func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 t.Run("the imports a build answers reach the done key and the log", importsABuildAnswersReachTheDoneKeyAndTheLog) + t.Run("a kept build names its node in its end", aKeptBuildNamesItsNodeInItsEnd) t.Run("the out-of-step files a build answers reach the log", outOfStepFilesABuildAnswersReachTheLog) t.Run("ADR 0624: a done key keeps each form a counted node reads", aDoneKeyKeepsEachFormACountedNodeReads) r := newRig(t, goBundle, archiveA, tsBundle) From c7ad2477f0436fb9c33f07512b3e1d2c5cc692cc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:26:55 +0100 Subject: [PATCH 4/9] BL6 (key codec): the catalog key's codec is meshcatalogkey, so the builder's builds package reaches no bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit meshcatalogclient imports the new package and keeps two one-line wrappers for the callers of the other parts until they move to it. Builds: design 47 G §670, §614 (wiring unchanged) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../clients/meshcatalogclient/fingerprint.go | 27 +++---------- contracts/go/clients/meshcatalogclient/go.mod | 1 + contracts/go/clients/meshcatalogkey/go.mod | 5 +++ contracts/go/clients/meshcatalogkey/key.go | 32 +++++++++++++++ .../go/clients/meshcatalogkey/key_test.go | 39 +++++++++++++++++++ modules/mesh-delivery-builder/builds/go.mod | 2 +- 6 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 contracts/go/clients/meshcatalogkey/go.mod create mode 100644 contracts/go/clients/meshcatalogkey/key.go create mode 100644 contracts/go/clients/meshcatalogkey/key_test.go diff --git a/contracts/go/clients/meshcatalogclient/fingerprint.go b/contracts/go/clients/meshcatalogclient/fingerprint.go index 0565fdb5..6ed3c43a 100644 --- a/contracts/go/clients/meshcatalogclient/fingerprint.go +++ b/contracts/go/clients/meshcatalogclient/fingerprint.go @@ -5,14 +5,11 @@ package meshcatalogclient import ( - "fmt" - "strings" - - "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey" ) // SourcePrefix2 opens every source fingerprint of the form. -const SourcePrefix2 = "src2:" +const SourcePrefix2 = meshcatalogkey.SourcePrefix2 // Toolchain2 is what an artifact's toolchain enters the fingerprint as (novox/hq ADR 0405 clause 7): the artifact's // language, the installed toolchain's version and the build module's recipe version. @@ -22,20 +19,8 @@ type Toolchain2 struct { Recipe string } -// KeyPart is a fingerprint as a key part: its 64 hex digits without the prefix, since a bus key may not hold ':'. It -// refuses a value that is not `src2:` and 64 lowercase hex digits, a src1 fingerprint included, naming it. -func KeyPart(fp string) (string, error) { - part, ok := strings.CutPrefix(fp, SourcePrefix2) - if !ok || !keys.IsHex64(part) { - return "", fmt.Errorf("fingerprint: %q is not a fingerprint of the form, %s and 64 lowercase hex digits", fp, SourcePrefix2) - } - return part, nil -} +// KeyPart is meshcatalogkey.KeyPart, the codec of a catalog key's parts (design 47 G §670). +func KeyPart(fp string) (string, error) { return meshcatalogkey.KeyPart(fp) } -// ParseKeyPart is the inverse of KeyPart. It refuses a part that is not 64 lowercase hex digits, naming it. -func ParseKeyPart(part string) (string, error) { - if !keys.IsHex64(part) { - return "", fmt.Errorf("fingerprint: %q is not the key part of a fingerprint, 64 lowercase hex digits", part) - } - return SourcePrefix2 + part, nil -} +// ParseKeyPart is meshcatalogkey.ParseKeyPart. +func ParseKeyPart(part string) (string, error) { return meshcatalogkey.ParseKeyPart(part) } diff --git a/contracts/go/clients/meshcatalogclient/go.mod b/contracts/go/clients/meshcatalogclient/go.mod index 433b7918..8a0a9aa3 100644 --- a/contracts/go/clients/meshcatalogclient/go.mod +++ b/contracts/go/clients/meshcatalogclient/go.mod @@ -3,6 +3,7 @@ module git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient go 1.26.0 require ( + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 ) diff --git a/contracts/go/clients/meshcatalogkey/go.mod b/contracts/go/clients/meshcatalogkey/go.mod new file mode 100644 index 00000000..e6b41bd8 --- /dev/null +++ b/contracts/go/clients/meshcatalogkey/go.mod @@ -0,0 +1,5 @@ +module git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/contracts/go/clients/meshcatalogkey/key.go b/contracts/go/clients/meshcatalogkey/key.go new file mode 100644 index 00000000..17d65d55 --- /dev/null +++ b/contracts/go/clients/meshcatalogkey/key.go @@ -0,0 +1,32 @@ +// Package meshcatalogkey is the codec of a catalog key's parts (design 47 G §670): a fingerprint as the part of a bus +// key, and back. It imports neither mesh-sdk's views nor the bus client, so a package that only reads and writes keys +// (the builder's builds package) reaches no bus; meshcatalogclient imports it. +package meshcatalogkey + +import ( + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// SourcePrefix2 is the prefix of a fingerprint of the current form. +const SourcePrefix2 = "src2:" + +// KeyPart is a fingerprint as a key part: its 64 hex digits without the prefix, since a bus key may not hold ':'. It +// refuses a value that is not `src2:` and 64 lowercase hex digits, a src1 fingerprint included, naming it. +func KeyPart(fp string) (string, error) { + part, ok := strings.CutPrefix(fp, SourcePrefix2) + if !ok || !keys.IsHex64(part) { + return "", fmt.Errorf("fingerprint: %q is not a fingerprint of the form, %s and 64 lowercase hex digits", fp, SourcePrefix2) + } + return part, nil +} + +// ParseKeyPart is the inverse of KeyPart. It refuses a part that is not 64 lowercase hex digits, naming it. +func ParseKeyPart(part string) (string, error) { + if !keys.IsHex64(part) { + return "", fmt.Errorf("fingerprint: %q is not the key part of a fingerprint, 64 lowercase hex digits", part) + } + return SourcePrefix2 + part, nil +} diff --git a/contracts/go/clients/meshcatalogkey/key_test.go b/contracts/go/clients/meshcatalogkey/key_test.go new file mode 100644 index 00000000..f8051de5 --- /dev/null +++ b/contracts/go/clients/meshcatalogkey/key_test.go @@ -0,0 +1,39 @@ +package meshcatalogkey + +import ( + "go/build" + "strings" + "testing" +) + +func TestKeyPartRoundTripsAndRefusesOthers(t *testing.T) { + fp := "src2:" + strings.Repeat("a", 64) + part, err := KeyPart(fp) + if err != nil || part != strings.Repeat("a", 64) { + t.Fatalf("KeyPart = %q, %v", part, err) + } + if back, err := ParseKeyPart(part); err != nil || back != fp { + t.Fatalf("ParseKeyPart = %q, %v", back, err) + } + for _, bad := range []string{"src1:" + strings.Repeat("a", 64), "src2:abc", strings.Repeat("a", 64)} { + if _, err := KeyPart(bad); err == nil { + t.Errorf("KeyPart(%q) accepted", bad) + } + } + if _, err := ParseKeyPart("abc"); err == nil { + t.Error("ParseKeyPart(abc) accepted") + } +} + +// Design 47 G §670: the package imports no bus. +func TestThePackageImportsNoBus(t *testing.T) { + p, err := build.ImportDir(".", 0) + if err != nil { + t.Fatal(err) + } + for _, i := range p.Imports { + if strings.Contains(i, "nats") || strings.Contains(i, "/views") { + t.Errorf("meshcatalogkey imports %s", i) + } + } +} diff --git a/modules/mesh-delivery-builder/builds/go.mod b/modules/mesh-delivery-builder/builds/go.mod index 84dbe8cb..4fb85e3c 100644 --- a/modules/mesh-delivery-builder/builds/go.mod +++ b/modules/mesh-delivery-builder/builds/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 From 7ecae5a6dd6149255f8a230aec1d3d06298a7fa4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 16:01:06 +0100 Subject: [PATCH 5/9] CT12: the build seats' ask carries its version, every ask says ask-taken and step-ended (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's reserved version reaches the build in the ask; a holder says when it takes and ends any ask, a builder's build included, so the watchman times every ask; the toolchains key names its program's digest; the checker may drop its own asks until its removal. step-progress and ask-taken name no keep and go on the shared events stream. The delivery SDK's neutral forms follow the part. Builds: design 47 G §472, §548, §549, §552, §573, §602, §603 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- contracts/go/seats/meshwants/contract.go | 3 - contracts/go/seats/nodebuildgo/contract.go | 49 +++++++++--- .../go/seats/nodebuildgo/contract_test.go | 16 +++- contracts/go/seats/nodebuildimage/contract.go | 49 +++++++++--- .../go/seats/nodebuildimage/contract_test.go | 16 +++- contracts/go/seats/nodebuildnode/contract.go | 49 +++++++++--- .../go/seats/nodebuildnode/contract_test.go | 16 +++- contracts/parts/build-step.json | 75 +++++++++++++++---- contracts/seats/mesh-wants/seat.json | 4 - .../seats/node-build-go/fixtures/events.json | 36 ++++++++- .../seats/node-build-go/fixtures/states.json | 31 ++++++++ .../node-build-image/fixtures/events.json | 36 ++++++++- .../node-build-image/fixtures/states.json | 31 ++++++++ .../node-build-node/fixtures/events.json | 36 ++++++++- .../node-build-node/fixtures/states.json | 31 ++++++++ contracts/ts/seats/meshwants/index.d.ts | 2 - contracts/ts/seats/meshwants/index.js | 2 - contracts/ts/seats/nodebuildgo/index.d.ts | 9 ++- contracts/ts/seats/nodebuildgo/index.js | 9 ++- .../seats/nodebuildgo/parts/build-step.d.ts | 38 ++++++++-- .../ts/seats/nodebuildgo/parts/build-step.js | 1 + contracts/ts/seats/nodebuildimage/index.d.ts | 9 ++- contracts/ts/seats/nodebuildimage/index.js | 9 ++- .../nodebuildimage/parts/build-step.d.ts | 38 ++++++++-- .../seats/nodebuildimage/parts/build-step.js | 1 + contracts/ts/seats/nodebuildnode/index.d.ts | 9 ++- contracts/ts/seats/nodebuildnode/index.js | 9 ++- .../seats/nodebuildnode/parts/build-step.d.ts | 38 ++++++++-- .../seats/nodebuildnode/parts/build-step.js | 1 + sdk/buildstep/ask.go | 21 +++--- sdk/buildstep/ask_test.go | 23 +++++- sdk/buildstep/pipelinestep.go | 35 +++++++-- sdk/buildstep/toolchain.go | 13 ++-- sdk/buildstep/toolchain_test.go | 4 +- 34 files changed, 622 insertions(+), 127 deletions(-) diff --git a/contracts/go/seats/meshwants/contract.go b/contracts/go/seats/meshwants/contract.go index ad44c028..f4cb11ff 100644 --- a/contracts/go/seats/meshwants/contract.go +++ b/contracts/go/seats/meshwants/contract.go @@ -100,9 +100,6 @@ const StreamVersionDelivered = "SEAT_mesh-wants_version-delivered" // SubjectVersionDelivered is where version-delivered is published. const SubjectVersionDelivered = "mesh.seat.mesh-wants.event.version-delivered" -// StreamStepProgress is the stream step-progress is kept in. -const StreamStepProgress = "SEAT_mesh-wants_step-progress" - // SubjectStepProgress is where step-progress is published, on mesh.seat.mesh-wants.event.step-progress.. func SubjectStepProgress(node string) string { return "mesh.seat.mesh-wants.event.step-progress." + node diff --git a/contracts/go/seats/nodebuildgo/contract.go b/contracts/go/seats/nodebuildgo/contract.go index d1ae0f25..7db4425a 100644 --- a/contracts/go/seats/nodebuildgo/contract.go +++ b/contracts/go/seats/nodebuildgo/contract.go @@ -36,6 +36,7 @@ const ( EventStepEnded = "step-ended" EventToolchainReported = "toolchain-reported" EventStepProgress = "step-progress" + EventAskTaken = "ask-taken" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -45,7 +46,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded, EventToolchainReported, EventStepProgress} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress, EventAskTaken} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -74,9 +75,6 @@ func SubjectToolchainReported(node string) string { // SubjectToolchainReportedAll hears every toolchain-reported. const SubjectToolchainReportedAll = "mesh.seat.node-build-go.event.toolchain-reported.*" -// StreamStepProgress is the stream step-progress is kept in. -const StreamStepProgress = "SEAT_node-build-go_step-progress" - // SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-go.event.step-progress.. func SubjectStepProgress(node string) string { return "mesh.seat.node-build-go.event.step-progress." + node @@ -85,6 +83,12 @@ func SubjectStepProgress(node string) string { // SubjectStepProgressAll hears every step-progress. const SubjectStepProgressAll = "mesh.seat.node-build-go.event.step-progress.*" +// SubjectAskTaken is where ask-taken is published, on mesh.seat.node-build-go.event.ask-taken.. +func SubjectAskTaken(node string) string { return "mesh.seat.node-build-go.event.ask-taken." + node } + +// SubjectAskTakenAll hears every ask-taken. +const SubjectAskTakenAll = "mesh.seat.node-build-go.event.ask-taken.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-go.accept.delivery" @@ -372,6 +376,16 @@ type AskArtifact struct { Version string `json:"version,omitempty"` } +// AskTaken is an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602). +type AskTaken struct { + // the ask's id + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + // the node that took it, as in the subject + Node string `json:"node"` +} + // BuildAnswer is a type of the contract. type BuildAnswer struct { Artifacts []BuiltArtifact `json:"artifacts"` @@ -761,23 +775,31 @@ type StepAsk struct { Step *StepRun `json:"step,omitempty"` // the layout version V int64 `json:"v,omitempty"` + // the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573) + Version string `json:"version,omitempty"` } -// StepEnded is a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at. +// StepEnded is an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548). type StepEnded struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the ask's id, by which a reader takes only its own asks + Ask string `json:"ask"` + // who asked it + Asker string `json:"asker"` + // when it ended (RFC 3339 time) + At time.Time `json:"at"` // for a rerun, the caller the runtime proved By string `json:"by,omitempty"` // the packages not run because the ask's kept results held them Carried []KeptResult `json:"carried,omitempty"` - Commit string `json:"commit"` + Commit string `json:"commit,omitempty"` Ended *time.Time `json:"ended,omitempty"` - Fingerprint string `json:"fingerprint"` + Fingerprint string `json:"fingerprint,omitempty"` // the ask's id ID string `json:"id"` // the packages that passed at their digest, for the runner to keep Kept []KeptResult `json:"kept,omitempty"` - Module string `json:"module"` + Module string `json:"module,omitempty"` // the node that ran it, as in the subject Node string `json:"node"` // never passed for a step that ran nothing @@ -786,12 +808,12 @@ type StepEnded struct { Output string `json:"output,omitempty"` // the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) OutputObject string `json:"output-object,omitempty"` - Pipeline string `json:"pipeline"` + Pipeline string `json:"pipeline,omitempty"` Rerun bool `json:"rerun,omitempty"` // why it failed or could not run, naming the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` - Step string `json:"step"` + Step string `json:"step,omitempty"` Tests *TestCounts `json:"tests,omitempty"` // the layout version V int64 `json:"v,omitempty"` @@ -912,6 +934,8 @@ type TestCounts struct { type Toolchain struct { // when (RFC 3339 time) At *time.Time `json:"at,omitempty"` + // the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472) + Digest string `json:"digest,omitempty"` // go or typescript Language string `json:"language"` Recipe string `json:"recipe"` @@ -1067,6 +1091,9 @@ type ToolchainReportedEvent = ToolchainReported // StepProgressEvent is the body of the event step-progress. type StepProgressEvent = StepProgress +// AskTakenEvent is the body of the event ask-taken. +type AskTakenEvent = AskTaken + // StepsByAskerNodeIDValue is one value of the state steps under the key ... type StepsByAskerNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildgo/contract_test.go b/contracts/go/seats/nodebuildgo/contract_test.go index 1948bb46..50eb29a1 100644 --- a/contracts/go/seats/nodebuildgo/contract_test.go +++ b/contracts/go/seats/nodebuildgo/contract_test.go @@ -17,14 +17,18 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a unit step ended passed on its node, with its counts and what it carried", subject: "mesh.seat.node-build-go.event.step-ended.ace", given: "{\"artifact\":\"nats:program\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", + {name: "a unit step ended passed on its node, with its counts and what it carried", subject: "mesh.seat.node-build-go.event.step-ended.ace", given: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, - {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-go.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", + {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-go.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5f\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5f\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a node's toolchain reported", subject: "mesh.seat.node-build-go.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.toolchain-reported\\.[^.*> ]+$")}, {name: "a running ask says its progress", subject: "mesh.seat.node-build-go.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-progress\\.[^.*> ]+$")}, + {name: "a builder's build ended, naming its asker and no commit or fingerprint (G 548)", subject: "mesh.seat.node-build-go.event.step-ended.ace", given: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:03:00Z\",\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"ace\",\"outcome\":\"failed\",\"said\":\"the build failed\",\"v\":1}", wire: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:03:00Z\",\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"ace\",\"outcome\":\"failed\",\"said\":\"the build failed\",\"v\":1}", + decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "an ask taken, whatever its asker (G 602)", subject: "mesh.seat.node-build-go.event.ask-taken.ace", given: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:00:00Z\",\"node\":\"ace\"}", wire: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:00:00Z\",\"node\":\"ace\"}", + decode: func(b []byte) (any, error) { var v AskTakenEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.ask-taken\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -45,8 +49,16 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a node's toolchain with its program's digest (G 472)", subject: "", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"recipe\":\"1\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"recipe\":\"1\",\"v\":1,\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { + var v ToolchainsByNodeLanguageValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "a build ask", subject: "mesh.seat.node-build-go.accept.delivery", given: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b2\",\"kind\":\"build\",\"language\":\"go\"}", wire: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b2\",\"kind\":\"build\",\"language\":\"go\"}", decode: func(b []byte) (any, error) { var v DeliveryAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.accept\\.delivery$")}, + {name: "a build ask with the version the builder reserved (G 573)", subject: "mesh.seat.node-build-go.accept.delivery", given: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b3\",\"kind\":\"build\",\"language\":\"go\",\"version\":\"1.4.2\"}", wire: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b3\",\"kind\":\"build\",\"language\":\"go\",\"version\":\"1.4.2\"}", + decode: func(b []byte) (any, error) { var v DeliveryAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.accept\\.delivery$")}, {name: "a check ask with its fetch, siblings and levels", subject: "mesh.seat.node-build-go.accept.check", given: "{\"asker\":\"mesh-delivery-checker\",\"at\":\"2026-11-05T10:00:00Z\",\"check\":{\"fetch\":{\"base\":\"89abcdef0123456789abcdef0123456789abcdef\",\"beside\":[{\"commit\":\"fedcba9876543210fedcba9876543210fedcba98\",\"repository\":\"novox/mesh-sdk\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"packages\":{\"git.novox.be/novox/mesh-sdk/go\":\"fedcba9876543210fedcba9876543210fedcba98\"},\"ref\":\"refs/pull/7/head\",\"repository\":\"novox/mesh-catalog\"},\"kept\":true,\"levels\":[\"unit\",\"integration\"],\"node-names\":[\"ace\",\"g14\"],\"toolchain\":{\"language\":\"go\",\"recipe\":\"2\",\"version\":\"1.26.0\"}},\"id\":\"c3d4\",\"kind\":\"check\",\"language\":\"go\",\"v\":1}", wire: "{\"asker\":\"mesh-delivery-checker\",\"at\":\"2026-11-05T10:00:00Z\",\"check\":{\"fetch\":{\"base\":\"89abcdef0123456789abcdef0123456789abcdef\",\"beside\":[{\"commit\":\"fedcba9876543210fedcba9876543210fedcba98\",\"repository\":\"novox/mesh-sdk\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"packages\":{\"git.novox.be/novox/mesh-sdk/go\":\"fedcba9876543210fedcba9876543210fedcba98\"},\"ref\":\"refs/pull/7/head\",\"repository\":\"novox/mesh-catalog\"},\"kept\":true,\"levels\":[\"unit\",\"integration\"],\"node-names\":[\"ace\",\"g14\"],\"toolchain\":{\"language\":\"go\",\"recipe\":\"2\",\"version\":\"1.26.0\"}},\"id\":\"c3d4\",\"kind\":\"check\",\"language\":\"go\",\"v\":1}", decode: func(b []byte) (any, error) { var v CheckAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.accept\\.check$")}, {name: "a finished build step with its artifacts", subject: "", given: "{\"ended\":\"2026-11-05T10:03:00Z\",\"id\":\"a1b2\",\"node\":\"ace\",\"result\":{\"artifacts\":[{\"artifact\":\"go\",\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"reference\":\"container-image-registry/nats/go@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}],\"imports\":[\"git.novox.be/novox/mesh-sdk/go\"],\"toolchain\":{\"language\":\"go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}},\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"passed\",\"tool\":\"build\",\"v\":1}", wire: "{\"ended\":\"2026-11-05T10:03:00Z\",\"id\":\"a1b2\",\"node\":\"ace\",\"result\":{\"artifacts\":[{\"artifact\":\"go\",\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"reference\":\"container-image-registry/nats/go@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}],\"imports\":[\"git.novox.be/novox/mesh-sdk/go\"],\"toolchain\":{\"language\":\"go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}},\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"passed\",\"tool\":\"build\",\"v\":1}", diff --git a/contracts/go/seats/nodebuildimage/contract.go b/contracts/go/seats/nodebuildimage/contract.go index bde198f4..367353bb 100644 --- a/contracts/go/seats/nodebuildimage/contract.go +++ b/contracts/go/seats/nodebuildimage/contract.go @@ -35,6 +35,7 @@ const ( EventStepEnded = "step-ended" EventToolchainReported = "toolchain-reported" EventStepProgress = "step-progress" + EventAskTaken = "ask-taken" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -44,7 +45,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded, EventToolchainReported, EventStepProgress} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress, EventAskTaken} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -75,9 +76,6 @@ func SubjectToolchainReported(node string) string { // SubjectToolchainReportedAll hears every toolchain-reported. const SubjectToolchainReportedAll = "mesh.seat.node-build-image.event.toolchain-reported.*" -// StreamStepProgress is the stream step-progress is kept in. -const StreamStepProgress = "SEAT_node-build-image_step-progress" - // SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-image.event.step-progress.. func SubjectStepProgress(node string) string { return "mesh.seat.node-build-image.event.step-progress." + node @@ -86,6 +84,12 @@ func SubjectStepProgress(node string) string { // SubjectStepProgressAll hears every step-progress. const SubjectStepProgressAll = "mesh.seat.node-build-image.event.step-progress.*" +// SubjectAskTaken is where ask-taken is published, on mesh.seat.node-build-image.event.ask-taken.. +func SubjectAskTaken(node string) string { return "mesh.seat.node-build-image.event.ask-taken." + node } + +// SubjectAskTakenAll hears every ask-taken. +const SubjectAskTakenAll = "mesh.seat.node-build-image.event.ask-taken.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-image.accept.delivery" @@ -370,6 +374,16 @@ type AskArtifact struct { Version string `json:"version,omitempty"` } +// AskTaken is an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602). +type AskTaken struct { + // the ask's id + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + // the node that took it, as in the subject + Node string `json:"node"` +} + // BuildAnswer is a type of the contract. type BuildAnswer struct { Artifacts []BuiltArtifact `json:"artifacts"` @@ -759,23 +773,31 @@ type StepAsk struct { Step *StepRun `json:"step,omitempty"` // the layout version V int64 `json:"v,omitempty"` + // the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573) + Version string `json:"version,omitempty"` } -// StepEnded is a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at. +// StepEnded is an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548). type StepEnded struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the ask's id, by which a reader takes only its own asks + Ask string `json:"ask"` + // who asked it + Asker string `json:"asker"` + // when it ended (RFC 3339 time) + At time.Time `json:"at"` // for a rerun, the caller the runtime proved By string `json:"by,omitempty"` // the packages not run because the ask's kept results held them Carried []KeptResult `json:"carried,omitempty"` - Commit string `json:"commit"` + Commit string `json:"commit,omitempty"` Ended *time.Time `json:"ended,omitempty"` - Fingerprint string `json:"fingerprint"` + Fingerprint string `json:"fingerprint,omitempty"` // the ask's id ID string `json:"id"` // the packages that passed at their digest, for the runner to keep Kept []KeptResult `json:"kept,omitempty"` - Module string `json:"module"` + Module string `json:"module,omitempty"` // the node that ran it, as in the subject Node string `json:"node"` // never passed for a step that ran nothing @@ -784,12 +806,12 @@ type StepEnded struct { Output string `json:"output,omitempty"` // the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) OutputObject string `json:"output-object,omitempty"` - Pipeline string `json:"pipeline"` + Pipeline string `json:"pipeline,omitempty"` Rerun bool `json:"rerun,omitempty"` // why it failed or could not run, naming the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` - Step string `json:"step"` + Step string `json:"step,omitempty"` Tests *TestCounts `json:"tests,omitempty"` // the layout version V int64 `json:"v,omitempty"` @@ -910,6 +932,8 @@ type TestCounts struct { type Toolchain struct { // when (RFC 3339 time) At *time.Time `json:"at,omitempty"` + // the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472) + Digest string `json:"digest,omitempty"` // go or typescript Language string `json:"language"` Recipe string `json:"recipe"` @@ -1062,6 +1086,9 @@ type ToolchainReportedEvent = ToolchainReported // StepProgressEvent is the body of the event step-progress. type StepProgressEvent = StepProgress +// AskTakenEvent is the body of the event ask-taken. +type AskTakenEvent = AskTaken + // StepsByAskerNodeIDValue is one value of the state steps under the key ... type StepsByAskerNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildimage/contract_test.go b/contracts/go/seats/nodebuildimage/contract_test.go index 21300d5e..5ebe76a8 100644 --- a/contracts/go/seats/nodebuildimage/contract_test.go +++ b/contracts/go/seats/nodebuildimage/contract_test.go @@ -17,14 +17,18 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a lint step ended passed on its node", subject: "mesh.seat.node-build-image.event.step-ended.ace", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"lint\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"lint\",\"v\":1}", + {name: "a lint step ended passed on its node", subject: "mesh.seat.node-build-image.event.step-ended.ace", given: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"lint\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"lint\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, - {name: "a lint step that could not run, naming the part not heard", subject: "mesh.seat.node-build-image.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", + {name: "a lint step that could not run, naming the part not heard", subject: "mesh.seat.node-build-image.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5f\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5f\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a node's toolchain reported", subject: "mesh.seat.node-build-image.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.toolchain-reported\\.[^.*> ]+$")}, {name: "a running ask says its progress", subject: "mesh.seat.node-build-image.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-progress\\.[^.*> ]+$")}, + {name: "a builder's build ended, naming its asker and no commit or fingerprint (G 548)", subject: "mesh.seat.node-build-image.event.step-ended.ace", given: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:03:00Z\",\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"ace\",\"outcome\":\"failed\",\"said\":\"the build failed\",\"v\":1}", wire: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:03:00Z\",\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"ace\",\"outcome\":\"failed\",\"said\":\"the build failed\",\"v\":1}", + decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "an ask taken, whatever its asker (G 602)", subject: "mesh.seat.node-build-image.event.ask-taken.ace", given: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:00:00Z\",\"node\":\"ace\"}", wire: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:00:00Z\",\"node\":\"ace\"}", + decode: func(b []byte) (any, error) { var v AskTakenEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.ask-taken\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -45,8 +49,16 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a node's toolchain with its program's digest (G 472)", subject: "", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"recipe\":\"1\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"recipe\":\"1\",\"v\":1,\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { + var v ToolchainsByNodeLanguageValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "a build ask", subject: "mesh.seat.node-build-image.accept.delivery", given: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b2\",\"kind\":\"build\",\"language\":\"go\"}", wire: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b2\",\"kind\":\"build\",\"language\":\"go\"}", decode: func(b []byte) (any, error) { var v DeliveryAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.accept\\.delivery$")}, + {name: "a build ask with the version the builder reserved (G 573)", subject: "mesh.seat.node-build-image.accept.delivery", given: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b3\",\"kind\":\"build\",\"language\":\"go\",\"version\":\"1.4.2\"}", wire: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b3\",\"kind\":\"build\",\"language\":\"go\",\"version\":\"1.4.2\"}", + decode: func(b []byte) (any, error) { var v DeliveryAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.accept\\.delivery$")}, {name: "a step ended on a transport fault, given again by the bus", subject: "", given: "{\"again-at\":\"2026-11-05T10:01:30Z\",\"ended\":\"2026-11-05T10:00:30Z\",\"id\":\"a1b2\",\"node\":\"ace\",\"passing\":true,\"reason\":\"fetching the commit: kex_exchange_identification: read: Connection reset by peer\",\"run\":1,\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"again\",\"tool\":\"build\",\"v\":1}", wire: "{\"again-at\":\"2026-11-05T10:01:30Z\",\"ended\":\"2026-11-05T10:00:30Z\",\"id\":\"a1b2\",\"node\":\"ace\",\"passing\":true,\"reason\":\"fetching the commit: kex_exchange_identification: read: Connection reset by peer\",\"run\":1,\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"again\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "an ask dropped by the caller that no longer waits", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"mesh-delivery-builder\",\"v\":1,\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"mesh-delivery-builder\",\"v\":1,\"why\":\"a newer fingerprint supersedes it\"}", diff --git a/contracts/go/seats/nodebuildnode/contract.go b/contracts/go/seats/nodebuildnode/contract.go index edd7230d..6014fe4a 100644 --- a/contracts/go/seats/nodebuildnode/contract.go +++ b/contracts/go/seats/nodebuildnode/contract.go @@ -35,6 +35,7 @@ const ( EventStepEnded = "step-ended" EventToolchainReported = "toolchain-reported" EventStepProgress = "step-progress" + EventAskTaken = "ask-taken" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -44,7 +45,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded, EventToolchainReported, EventStepProgress} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress, EventAskTaken} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -75,9 +76,6 @@ func SubjectToolchainReported(node string) string { // SubjectToolchainReportedAll hears every toolchain-reported. const SubjectToolchainReportedAll = "mesh.seat.node-build-node.event.toolchain-reported.*" -// StreamStepProgress is the stream step-progress is kept in. -const StreamStepProgress = "SEAT_node-build-node_step-progress" - // SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-node.event.step-progress.. func SubjectStepProgress(node string) string { return "mesh.seat.node-build-node.event.step-progress." + node @@ -86,6 +84,12 @@ func SubjectStepProgress(node string) string { // SubjectStepProgressAll hears every step-progress. const SubjectStepProgressAll = "mesh.seat.node-build-node.event.step-progress.*" +// SubjectAskTaken is where ask-taken is published, on mesh.seat.node-build-node.event.ask-taken.. +func SubjectAskTaken(node string) string { return "mesh.seat.node-build-node.event.ask-taken." + node } + +// SubjectAskTakenAll hears every ask-taken. +const SubjectAskTakenAll = "mesh.seat.node-build-node.event.ask-taken.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-node.accept.delivery" @@ -370,6 +374,16 @@ type AskArtifact struct { Version string `json:"version,omitempty"` } +// AskTaken is an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602). +type AskTaken struct { + // the ask's id + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + // the node that took it, as in the subject + Node string `json:"node"` +} + // BuildAnswer is a type of the contract. type BuildAnswer struct { Artifacts []BuiltArtifact `json:"artifacts"` @@ -759,23 +773,31 @@ type StepAsk struct { Step *StepRun `json:"step,omitempty"` // the layout version V int64 `json:"v,omitempty"` + // the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573) + Version string `json:"version,omitempty"` } -// StepEnded is a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at. +// StepEnded is an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548). type StepEnded struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the ask's id, by which a reader takes only its own asks + Ask string `json:"ask"` + // who asked it + Asker string `json:"asker"` + // when it ended (RFC 3339 time) + At time.Time `json:"at"` // for a rerun, the caller the runtime proved By string `json:"by,omitempty"` // the packages not run because the ask's kept results held them Carried []KeptResult `json:"carried,omitempty"` - Commit string `json:"commit"` + Commit string `json:"commit,omitempty"` Ended *time.Time `json:"ended,omitempty"` - Fingerprint string `json:"fingerprint"` + Fingerprint string `json:"fingerprint,omitempty"` // the ask's id ID string `json:"id"` // the packages that passed at their digest, for the runner to keep Kept []KeptResult `json:"kept,omitempty"` - Module string `json:"module"` + Module string `json:"module,omitempty"` // the node that ran it, as in the subject Node string `json:"node"` // never passed for a step that ran nothing @@ -784,12 +806,12 @@ type StepEnded struct { Output string `json:"output,omitempty"` // the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) OutputObject string `json:"output-object,omitempty"` - Pipeline string `json:"pipeline"` + Pipeline string `json:"pipeline,omitempty"` Rerun bool `json:"rerun,omitempty"` // why it failed or could not run, naming the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` - Step string `json:"step"` + Step string `json:"step,omitempty"` Tests *TestCounts `json:"tests,omitempty"` // the layout version V int64 `json:"v,omitempty"` @@ -910,6 +932,8 @@ type TestCounts struct { type Toolchain struct { // when (RFC 3339 time) At *time.Time `json:"at,omitempty"` + // the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472) + Digest string `json:"digest,omitempty"` // go or typescript Language string `json:"language"` Recipe string `json:"recipe"` @@ -1062,6 +1086,9 @@ type ToolchainReportedEvent = ToolchainReported // StepProgressEvent is the body of the event step-progress. type StepProgressEvent = StepProgress +// AskTakenEvent is the body of the event ask-taken. +type AskTakenEvent = AskTaken + // StepsByAskerNodeIDValue is one value of the state steps under the key ... type StepsByAskerNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildnode/contract_test.go b/contracts/go/seats/nodebuildnode/contract_test.go index 3eb5ebe0..78aff7ee 100644 --- a/contracts/go/seats/nodebuildnode/contract_test.go +++ b/contracts/go/seats/nodebuildnode/contract_test.go @@ -17,14 +17,18 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a unit step ended passed on its node, with its counts and what it carried", subject: "mesh.seat.node-build-node.event.step-ended.ace", given: "{\"artifact\":\"nats:program\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", + {name: "a unit step ended passed on its node, with its counts and what it carried", subject: "mesh.seat.node-build-node.event.step-ended.ace", given: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"carried\":[{\"digest\":\"sha256:bbbb\",\"level\":\"unit\",\"package\":\"./modules/nats/a\"}],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"ended\":\"2026-11-10T10:02:00Z\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5e\",\"kept\":[{\"digest\":\"sha256:cccc\",\"level\":\"unit\",\"package\":\"./modules/nats/b\"}],\"module\":\"nats\",\"node\":\"ace\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"unit\",\"tests\":{\"unit\":14},\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, - {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-node.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", + {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-node.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5f\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5f\",\"asker\":\"mesh-delivery-pipeline\",\"at\":\"2026-11-10T10:02:00Z\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a node's toolchain reported", subject: "mesh.seat.node-build-node.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.toolchain-reported\\.[^.*> ]+$")}, {name: "a running ask says its progress", subject: "mesh.seat.node-build-node.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-progress\\.[^.*> ]+$")}, + {name: "a builder's build ended, naming its asker and no commit or fingerprint (G 548)", subject: "mesh.seat.node-build-node.event.step-ended.ace", given: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:03:00Z\",\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"ace\",\"outcome\":\"failed\",\"said\":\"the build failed\",\"v\":1}", wire: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:03:00Z\",\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"ace\",\"outcome\":\"failed\",\"said\":\"the build failed\",\"v\":1}", + decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "an ask taken, whatever its asker (G 602)", subject: "mesh.seat.node-build-node.event.ask-taken.ace", given: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:00:00Z\",\"node\":\"ace\"}", wire: "{\"ask\":\"m2kx9a1b3c4d6a\",\"asker\":\"mesh-delivery-builder\",\"at\":\"2026-11-11T10:00:00Z\",\"node\":\"ace\"}", + decode: func(b []byte) (any, error) { var v AskTakenEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.ask-taken\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -45,8 +49,16 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a node's toolchain with its program's digest (G 472)", subject: "", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"recipe\":\"1\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"recipe\":\"1\",\"v\":1,\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { + var v ToolchainsByNodeLanguageValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "a build ask", subject: "mesh.seat.node-build-node.accept.delivery", given: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b2\",\"kind\":\"build\",\"language\":\"go\"}", wire: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b2\",\"kind\":\"build\",\"language\":\"go\"}", decode: func(b []byte) (any, error) { var v DeliveryAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.accept\\.delivery$")}, + {name: "a build ask with the version the builder reserved (G 573)", subject: "mesh.seat.node-build-node.accept.delivery", given: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b3\",\"kind\":\"build\",\"language\":\"go\",\"version\":\"1.4.2\"}", wire: "{\"asker\":\"mesh-delivery-builder\",\"build\":{\"artifacts\":[{\"artifact\":\"go\"}],\"module\":\"nats\"},\"id\":\"a1b3\",\"kind\":\"build\",\"language\":\"go\",\"version\":\"1.4.2\"}", + decode: func(b []byte) (any, error) { var v DeliveryAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.accept\\.delivery$")}, {name: "a check ask with the group's tagged package", subject: "mesh.seat.node-build-node.accept.check", given: "{\"asker\":\"mesh-delivery-checker\",\"at\":\"2026-11-05T10:00:00Z\",\"check\":{\"fetch\":{\"base\":\"89abcdef0123456789abcdef0123456789abcdef\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"package\":{\"name\":\"@novox/mesh-sdk\",\"version\":\"0.1.15-feat-x.1\"},\"ref\":\"refs/pull/7/head\",\"repository\":\"novox/mesh-delivery\"},\"kept\":true,\"levels\":[\"unit\",\"integration\"],\"node-names\":[\"ace\",\"g14\"],\"toolchain\":{\"language\":\"typescript\",\"recipe\":\"1\",\"version\":\"26.12.0\"}},\"id\":\"c3d5\",\"kind\":\"check\",\"language\":\"typescript\",\"v\":1}", wire: "{\"asker\":\"mesh-delivery-checker\",\"at\":\"2026-11-05T10:00:00Z\",\"check\":{\"fetch\":{\"base\":\"89abcdef0123456789abcdef0123456789abcdef\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"package\":{\"name\":\"@novox/mesh-sdk\",\"version\":\"0.1.15-feat-x.1\"},\"ref\":\"refs/pull/7/head\",\"repository\":\"novox/mesh-delivery\"},\"kept\":true,\"levels\":[\"unit\",\"integration\"],\"node-names\":[\"ace\",\"g14\"],\"toolchain\":{\"language\":\"typescript\",\"recipe\":\"1\",\"version\":\"26.12.0\"}},\"id\":\"c3d5\",\"kind\":\"check\",\"language\":\"typescript\",\"v\":1}", decode: func(b []byte) (any, error) { var v CheckAccept; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.accept\\.check$")}, {name: "a finished check step of a TypeScript merge preview", subject: "", given: "{\"ended\":\"2026-11-05T10:01:00Z\",\"id\":\"c3d6\",\"node\":\"ace\",\"result\":{\"fetch\":{\"base\":\"abcdef0123456789abcdef0123456789abcdef01\",\"changed\":[\"modules/m/package.json\"],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"merge\":\"0123456789abcdef0123456789abcdef01234567\",\"modules\":[\"modules/m\"],\"trees\":{\"modules/m\":{\"manifest\":\"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef\",\"tree\":\"0123456789abcdef0123456789abcdef01234567\"}},\"workspace\":\"mvii1sxt979f67\"}},\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"passed\",\"tool\":\"check\",\"v\":1}", wire: "{\"ended\":\"2026-11-05T10:01:00Z\",\"id\":\"c3d6\",\"node\":\"ace\",\"result\":{\"fetch\":{\"base\":\"abcdef0123456789abcdef0123456789abcdef01\",\"changed\":[\"modules/m/package.json\"],\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"merge\":\"0123456789abcdef0123456789abcdef01234567\",\"modules\":[\"modules/m\"],\"trees\":{\"modules/m\":{\"manifest\":\"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef\",\"tree\":\"0123456789abcdef0123456789abcdef01234567\"}},\"workspace\":\"mvii1sxt979f67\"}},\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"passed\",\"tool\":\"check\",\"v\":1}", diff --git a/contracts/parts/build-step.json b/contracts/parts/build-step.json index 6b522354..8c0b00b6 100644 --- a/contracts/parts/build-step.json +++ b/contracts/parts/build-step.json @@ -44,7 +44,8 @@ "description": "End an ask this node holds for a caller that no longer waits for it: a step not yet running is ended dropped when the bus gives it, one running is cancelled; both end the step dropped, naming who dropped it and why. Writes dropped. in the pauses state; the build module removes the key when it has acted.", "callers": [ "seat:mesh-pipeline-runner", - "seat:mesh-builds" + "seat:mesh-builds", + "seat:mesh-delivery-checker" ], "input": { "type": "object", @@ -261,6 +262,10 @@ "description": "when (RFC 3339 time)", "format": "date-time" }, + "version": { + "type": "string", + "description": "the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573)" + }, "build": { "type": "object", "properties": { @@ -682,6 +687,10 @@ "type": "string", "description": "when (RFC 3339 time)", "format": "date-time" + }, + "digest": { + "type": "string", + "description": "the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472)" } }, "required": [ @@ -1508,7 +1517,7 @@ }, "StepEnded": { "type": "object", - "description": "a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at", + "description": "an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548)", "properties": { "v": { "type": "integer", @@ -1518,6 +1527,14 @@ "type": "string", "description": "the ask's id" }, + "asker": { + "type": "string", + "description": "who asked it" + }, + "ask": { + "type": "string", + "description": "the ask's id, by which a reader takes only its own asks" + }, "pipeline": { "type": "string" }, @@ -1598,18 +1615,20 @@ "ended": { "type": "string", "format": "date-time" + }, + "at": { + "type": "string", + "description": "when it ended (RFC 3339 time)", + "format": "date-time" } }, "required": [ "id", - "pipeline", - "step", - "module", - "artifact", - "commit", - "fingerprint", + "asker", + "ask", "node", - "outcome" + "outcome", + "at" ] }, "DetectedStep": { @@ -1721,6 +1740,33 @@ "phase", "at" ] + }, + "AskTaken": { + "type": "object", + "description": "an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string", + "description": "the ask's id" + }, + "node": { + "type": "string", + "description": "the node that took it, as in the subject" + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "node", + "at" + ] } }, "events": [ @@ -1751,10 +1797,13 @@ "subject": "step-progress.", "body": { "$ref": "#/defs/StepProgress" - }, - "keep": { - "max-age-days": 1, - "max-mib": 16 + } + }, + { + "name": "ask-taken", + "subject": "ask-taken.", + "body": { + "$ref": "#/defs/AskTaken" } } ] diff --git a/contracts/seats/mesh-wants/seat.json b/contracts/seats/mesh-wants/seat.json index 6e41ad9f..b2ce7f21 100644 --- a/contracts/seats/mesh-wants/seat.json +++ b/contracts/seats/mesh-wants/seat.json @@ -431,10 +431,6 @@ { "name": "step-progress", "subject": "step-progress.", - "keep": { - "max-age-days": 1, - "max-mib": 16 - }, "body": { "type": "object", "description": "a composition step's progress, said once a minute while the assigner runs it (design 47 G section 496)", diff --git a/contracts/seats/node-build-go/fixtures/events.json b/contracts/seats/node-build-go/fixtures/events.json index 1d3351e3..61773fa3 100644 --- a/contracts/seats/node-build-go/fixtures/events.json +++ b/contracts/seats/node-build-go/fixtures/events.json @@ -6,6 +6,8 @@ "given": { "v": 1, "id": "m2kx9a1b3c4d5e", + "asker": "mesh-delivery-pipeline", + "ask": "m2kx9a1b3c4d5e", "pipeline": "main.12.nats", "step": "unit", "module": "nats", @@ -32,7 +34,8 @@ } ], "started": "2026-11-10T10:00:00Z", - "ended": "2026-11-10T10:02:00Z" + "ended": "2026-11-10T10:02:00Z", + "at": "2026-11-10T10:02:00Z" } }, { @@ -42,6 +45,8 @@ "given": { "v": 1, "id": "m2kx9a1b3c4d5f", + "asker": "mesh-delivery-pipeline", + "ask": "m2kx9a1b3c4d5f", "pipeline": "main.12.nats", "step": "integration", "module": "nats", @@ -51,7 +56,8 @@ "node": "g14", "outcome": "could not run", "said": "the sandbox could not fetch novox/mesh-sdk", - "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" + "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f", + "at": "2026-11-10T10:02:00Z" } }, { @@ -76,5 +82,31 @@ "phase": "fetch", "at": "2026-11-11T10:01:00Z" } + }, + { + "name": "a builder's build ended, naming its asker and no commit or fingerprint (G 548)", + "of": "event:step-ended", + "subject": "mesh.seat.node-build-go.event.step-ended.ace", + "given": { + "v": 1, + "id": "m2kx9a1b3c4d6a", + "asker": "mesh-delivery-builder", + "ask": "m2kx9a1b3c4d6a", + "node": "ace", + "outcome": "failed", + "said": "the build failed", + "at": "2026-11-11T10:03:00Z" + } + }, + { + "name": "an ask taken, whatever its asker (G 602)", + "of": "event:ask-taken", + "subject": "mesh.seat.node-build-go.event.ask-taken.ace", + "given": { + "asker": "mesh-delivery-builder", + "ask": "m2kx9a1b3c4d6a", + "node": "ace", + "at": "2026-11-11T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-go/fixtures/states.json b/contracts/seats/node-build-go/fixtures/states.json index 9794d3fa..98e3890d 100644 --- a/contracts/seats/node-build-go/fixtures/states.json +++ b/contracts/seats/node-build-go/fixtures/states.json @@ -41,6 +41,17 @@ "recipe": "1" } }, + { + "name": "a node's toolchain with its program's digest (G 472)", + "of": "state:toolchains/.", + "given": { + "v": 1, + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, { "name": "a build ask", "of": "accept:delivery", @@ -60,6 +71,26 @@ } } }, + { + "name": "a build ask with the version the builder reserved (G 573)", + "of": "accept:delivery", + "subject": "mesh.seat.node-build-go.accept.delivery", + "given": { + "id": "a1b3", + "kind": "build", + "language": "go", + "asker": "mesh-delivery-builder", + "version": "1.4.2", + "build": { + "module": "nats", + "artifacts": [ + { + "artifact": "go" + } + ] + } + } + }, { "name": "a check ask with its fetch, siblings and levels", "of": "accept:check", diff --git a/contracts/seats/node-build-image/fixtures/events.json b/contracts/seats/node-build-image/fixtures/events.json index e8dca58f..b4e9402d 100644 --- a/contracts/seats/node-build-image/fixtures/events.json +++ b/contracts/seats/node-build-image/fixtures/events.json @@ -6,6 +6,8 @@ "given": { "v": 1, "id": "m2kx9a1b3c4d5e", + "asker": "mesh-delivery-pipeline", + "ask": "m2kx9a1b3c4d5e", "pipeline": "main.12.nats", "step": "lint", "module": "nats", @@ -15,7 +17,8 @@ "node": "ace", "outcome": "passed", "started": "2026-11-10T10:00:00Z", - "ended": "2026-11-10T10:02:00Z" + "ended": "2026-11-10T10:02:00Z", + "at": "2026-11-10T10:02:00Z" } }, { @@ -25,6 +28,8 @@ "given": { "v": 1, "id": "m2kx9a1b3c4d5f", + "asker": "mesh-delivery-pipeline", + "ask": "m2kx9a1b3c4d5f", "pipeline": "main.12.nats", "step": "lint", "module": "nats", @@ -34,7 +39,8 @@ "node": "g14", "outcome": "could not run", "said": "the sandbox could not fetch novox/mesh-sdk", - "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" + "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f", + "at": "2026-11-10T10:02:00Z" } }, { @@ -59,5 +65,31 @@ "phase": "fetch", "at": "2026-11-11T10:01:00Z" } + }, + { + "name": "a builder's build ended, naming its asker and no commit or fingerprint (G 548)", + "of": "event:step-ended", + "subject": "mesh.seat.node-build-image.event.step-ended.ace", + "given": { + "v": 1, + "id": "m2kx9a1b3c4d6a", + "asker": "mesh-delivery-builder", + "ask": "m2kx9a1b3c4d6a", + "node": "ace", + "outcome": "failed", + "said": "the build failed", + "at": "2026-11-11T10:03:00Z" + } + }, + { + "name": "an ask taken, whatever its asker (G 602)", + "of": "event:ask-taken", + "subject": "mesh.seat.node-build-image.event.ask-taken.ace", + "given": { + "asker": "mesh-delivery-builder", + "ask": "m2kx9a1b3c4d6a", + "node": "ace", + "at": "2026-11-11T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-image/fixtures/states.json b/contracts/seats/node-build-image/fixtures/states.json index 9e60272a..f585ebf3 100644 --- a/contracts/seats/node-build-image/fixtures/states.json +++ b/contracts/seats/node-build-image/fixtures/states.json @@ -41,6 +41,17 @@ "recipe": "1" } }, + { + "name": "a node's toolchain with its program's digest (G 472)", + "of": "state:toolchains/.", + "given": { + "v": 1, + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, { "name": "a build ask", "of": "accept:delivery", @@ -60,6 +71,26 @@ } } }, + { + "name": "a build ask with the version the builder reserved (G 573)", + "of": "accept:delivery", + "subject": "mesh.seat.node-build-image.accept.delivery", + "given": { + "id": "a1b3", + "kind": "build", + "language": "go", + "asker": "mesh-delivery-builder", + "version": "1.4.2", + "build": { + "module": "nats", + "artifacts": [ + { + "artifact": "go" + } + ] + } + } + }, { "name": "a step ended on a transport fault, given again by the bus", "of": "state:steps/.", diff --git a/contracts/seats/node-build-node/fixtures/events.json b/contracts/seats/node-build-node/fixtures/events.json index 1f1924a0..aa0e8f98 100644 --- a/contracts/seats/node-build-node/fixtures/events.json +++ b/contracts/seats/node-build-node/fixtures/events.json @@ -6,6 +6,8 @@ "given": { "v": 1, "id": "m2kx9a1b3c4d5e", + "asker": "mesh-delivery-pipeline", + "ask": "m2kx9a1b3c4d5e", "pipeline": "main.12.nats", "step": "unit", "module": "nats", @@ -32,7 +34,8 @@ } ], "started": "2026-11-10T10:00:00Z", - "ended": "2026-11-10T10:02:00Z" + "ended": "2026-11-10T10:02:00Z", + "at": "2026-11-10T10:02:00Z" } }, { @@ -42,6 +45,8 @@ "given": { "v": 1, "id": "m2kx9a1b3c4d5f", + "asker": "mesh-delivery-pipeline", + "ask": "m2kx9a1b3c4d5f", "pipeline": "main.12.nats", "step": "integration", "module": "nats", @@ -51,7 +56,8 @@ "node": "g14", "outcome": "could not run", "said": "the sandbox could not fetch novox/mesh-sdk", - "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" + "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f", + "at": "2026-11-10T10:02:00Z" } }, { @@ -76,5 +82,31 @@ "phase": "fetch", "at": "2026-11-11T10:01:00Z" } + }, + { + "name": "a builder's build ended, naming its asker and no commit or fingerprint (G 548)", + "of": "event:step-ended", + "subject": "mesh.seat.node-build-node.event.step-ended.ace", + "given": { + "v": 1, + "id": "m2kx9a1b3c4d6a", + "asker": "mesh-delivery-builder", + "ask": "m2kx9a1b3c4d6a", + "node": "ace", + "outcome": "failed", + "said": "the build failed", + "at": "2026-11-11T10:03:00Z" + } + }, + { + "name": "an ask taken, whatever its asker (G 602)", + "of": "event:ask-taken", + "subject": "mesh.seat.node-build-node.event.ask-taken.ace", + "given": { + "asker": "mesh-delivery-builder", + "ask": "m2kx9a1b3c4d6a", + "node": "ace", + "at": "2026-11-11T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-node/fixtures/states.json b/contracts/seats/node-build-node/fixtures/states.json index af631c10..d57e12a1 100644 --- a/contracts/seats/node-build-node/fixtures/states.json +++ b/contracts/seats/node-build-node/fixtures/states.json @@ -41,6 +41,17 @@ "recipe": "1" } }, + { + "name": "a node's toolchain with its program's digest (G 472)", + "of": "state:toolchains/.", + "given": { + "v": 1, + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, { "name": "a build ask", "of": "accept:delivery", @@ -60,6 +71,26 @@ } } }, + { + "name": "a build ask with the version the builder reserved (G 573)", + "of": "accept:delivery", + "subject": "mesh.seat.node-build-node.accept.delivery", + "given": { + "id": "a1b3", + "kind": "build", + "language": "go", + "asker": "mesh-delivery-builder", + "version": "1.4.2", + "build": { + "module": "nats", + "artifacts": [ + { + "artifact": "go" + } + ] + } + } + }, { "name": "a check ask with the group's tagged package", "of": "accept:check", diff --git a/contracts/ts/seats/meshwants/index.d.ts b/contracts/ts/seats/meshwants/index.d.ts index 91a28584..b2b00598 100644 --- a/contracts/ts/seats/meshwants/index.d.ts +++ b/contracts/ts/seats/meshwants/index.d.ts @@ -62,8 +62,6 @@ export declare const versionDeliveredStream: "SEAT_mesh-wants_version-delivered" export declare const versionDeliveredSubject: "mesh.seat.mesh-wants.event.version-delivered"; -export declare const stepProgressStream: "SEAT_mesh-wants_step-progress"; - export declare function stepProgressSubject(node: string): string; export declare const stepProgressSubjectAll: "mesh.seat.mesh-wants.event.step-progress.*"; diff --git a/contracts/ts/seats/meshwants/index.js b/contracts/ts/seats/meshwants/index.js index 488f1228..df1d0f62 100644 --- a/contracts/ts/seats/meshwants/index.js +++ b/contracts/ts/seats/meshwants/index.js @@ -64,8 +64,6 @@ export const versionDeliveredStream = "SEAT_mesh-wants_version-delivered"; export const versionDeliveredSubject = "mesh.seat.mesh-wants.event.version-delivered"; -export const stepProgressStream = "SEAT_mesh-wants_step-progress"; - export function stepProgressSubject(node) { return "mesh.seat.mesh-wants.event.step-progress." + node; } diff --git a/contracts/ts/seats/nodebuildgo/index.d.ts b/contracts/ts/seats/nodebuildgo/index.d.ts index 11d2e419..1fe4b734 100644 --- a/contracts/ts/seats/nodebuildgo/index.d.ts +++ b/contracts/ts/seats/nodebuildgo/index.d.ts @@ -28,6 +28,7 @@ export declare const events: { readonly stepEnded: "step-ended"; readonly toolchainReported: "toolchain-reported"; readonly stepProgress: "step-progress"; + readonly askTaken: "ask-taken"; }; export declare const family: { readonly name: "node-build-go.events"; readonly maxAgeDays: 7 }; @@ -44,12 +45,14 @@ export declare function toolchainReportedSubject(node: string): string; export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-go.event.toolchain-reported.*"; -export declare const stepProgressStream: "SEAT_node-build-go_step-progress"; - export declare function stepProgressSubject(node: string): string; export declare const stepProgressSubjectAll: "mesh.seat.node-build-go.event.step-progress.*"; +export declare function askTakenSubject(node: string): string; + +export declare const askTakenSubjectAll: "mesh.seat.node-build-go.event.ask-taken.*"; + export declare const deliverySubject: "mesh.seat.node-build-go.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-go.accept.check"; @@ -112,6 +115,8 @@ export declare function detectedStepsByModuleArtifactKey(module: string, artifac export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { module: string; artifact: string } | undefined; export type AskArtifact = buildStep.AskArtifact; +export type AskTaken = buildStep.AskTaken; +export type AskTakenEvent = buildStep.AskTakenEvent; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; diff --git a/contracts/ts/seats/nodebuildgo/index.js b/contracts/ts/seats/nodebuildgo/index.js index cf4a3f99..f2566f56 100644 --- a/contracts/ts/seats/nodebuildgo/index.js +++ b/contracts/ts/seats/nodebuildgo/index.js @@ -26,6 +26,7 @@ export const events = { stepEnded: "step-ended", toolchainReported: "toolchain-reported", stepProgress: "step-progress", + askTaken: "ask-taken", }; export const family = { name: "node-build-go.events", maxAgeDays: 7 }; @@ -46,14 +47,18 @@ export function toolchainReportedSubject(node) { export const toolchainReportedSubjectAll = "mesh.seat.node-build-go.event.toolchain-reported.*"; -export const stepProgressStream = "SEAT_node-build-go_step-progress"; - export function stepProgressSubject(node) { return "mesh.seat.node-build-go.event.step-progress." + node; } export const stepProgressSubjectAll = "mesh.seat.node-build-go.event.step-progress.*"; +export function askTakenSubject(node) { + return "mesh.seat.node-build-go.event.ask-taken." + node; +} + +export const askTakenSubjectAll = "mesh.seat.node-build-go.event.ask-taken.*"; + export const deliverySubject = "mesh.seat.node-build-go.accept.delivery"; export const checkSubject = "mesh.seat.node-build-go.accept.check"; diff --git a/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index c8907fdb..2addf5c4 100644 --- a/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -15,6 +15,7 @@ export declare const events: { readonly stepEnded: "step-ended"; readonly toolchainReported: "toolchain-reported"; readonly stepProgress: "step-progress"; + readonly askTaken: "ask-taken"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -25,6 +26,16 @@ export interface AskArtifact { "version"?: string; } +/** an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602) */ +export interface AskTaken { + /** the ask's id */ + "ask": string; + "asker": string; + "at": string; + /** the node that took it, as in the subject */ + "node": string; +} + export interface BuildAnswer { "artifacts": BuiltArtifact[]; "imports"?: string[]; @@ -392,23 +403,31 @@ export interface StepAsk { "step"?: StepRun; /** the layout version */ "v"?: number; + /** the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573) */ + "version"?: string; } -/** a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at */ +/** an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548) */ export interface StepEnded { - "artifact": string; + "artifact"?: string; + /** the ask's id, by which a reader takes only its own asks */ + "ask": string; + /** who asked it */ + "asker": string; + /** when it ended (RFC 3339 time) */ + "at": string; /** for a rerun, the caller the runtime proved */ "by"?: string; /** the packages not run because the ask's kept results held them */ "carried"?: KeptResult[]; - "commit": string; + "commit"?: string; "ended"?: string; - "fingerprint": string; + "fingerprint"?: string; /** the ask's id */ "id": string; /** the packages that passed at their digest, for the runner to keep */ "kept"?: KeptResult[]; - "module": string; + "module"?: string; /** the node that ran it, as in the subject */ "node": string; /** never passed for a step that ran nothing */ @@ -417,12 +436,12 @@ export interface StepEnded { "output"?: string; /** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */ "output-object"?: string; - "pipeline": string; + "pipeline"?: string; "rerun"?: boolean; /** why it failed or could not run, naming the part not heard */ "said"?: string; "started"?: string; - "step": "lint" | "unit" | "integration"; + "step"?: "lint" | "unit" | "integration"; "tests"?: TestCounts; /** the layout version */ "v"?: number; @@ -513,6 +532,8 @@ export interface TestCounts { export interface Toolchain { /** when (RFC 3339 time) */ "at"?: string; + /** the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472) */ + "digest"?: string; /** go or typescript */ "language": string; "recipe": string; @@ -643,3 +664,6 @@ export type ToolchainReportedEvent = ToolchainReported; /** is the body of the event step-progress. */ export type StepProgressEvent = StepProgress; +/** is the body of the event ask-taken. */ +export type AskTakenEvent = AskTaken; + diff --git a/contracts/ts/seats/nodebuildgo/parts/build-step.js b/contracts/ts/seats/nodebuildgo/parts/build-step.js index 155a68be..400cdf1c 100644 --- a/contracts/ts/seats/nodebuildgo/parts/build-step.js +++ b/contracts/ts/seats/nodebuildgo/parts/build-step.js @@ -15,6 +15,7 @@ export const events = { stepEnded: "step-ended", toolchainReported: "toolchain-reported", stepProgress: "step-progress", + askTaken: "ask-taken", }; export {}; diff --git a/contracts/ts/seats/nodebuildimage/index.d.ts b/contracts/ts/seats/nodebuildimage/index.d.ts index 67b12889..dff95615 100644 --- a/contracts/ts/seats/nodebuildimage/index.d.ts +++ b/contracts/ts/seats/nodebuildimage/index.d.ts @@ -27,6 +27,7 @@ export declare const events: { readonly stepEnded: "step-ended"; readonly toolchainReported: "toolchain-reported"; readonly stepProgress: "step-progress"; + readonly askTaken: "ask-taken"; }; export declare const family: { readonly name: "node-build-image.events"; readonly maxAgeDays: 7 }; @@ -43,12 +44,14 @@ export declare function toolchainReportedSubject(node: string): string; export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-image.event.toolchain-reported.*"; -export declare const stepProgressStream: "SEAT_node-build-image_step-progress"; - export declare function stepProgressSubject(node: string): string; export declare const stepProgressSubjectAll: "mesh.seat.node-build-image.event.step-progress.*"; +export declare function askTakenSubject(node: string): string; + +export declare const askTakenSubjectAll: "mesh.seat.node-build-image.event.ask-taken.*"; + export declare const deliverySubject: "mesh.seat.node-build-image.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-image.accept.check"; @@ -109,6 +112,8 @@ export declare function detectedStepsByModuleArtifactKey(module: string, artifac export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { module: string; artifact: string } | undefined; export type AskArtifact = buildStep.AskArtifact; +export type AskTaken = buildStep.AskTaken; +export type AskTakenEvent = buildStep.AskTakenEvent; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; diff --git a/contracts/ts/seats/nodebuildimage/index.js b/contracts/ts/seats/nodebuildimage/index.js index 8a23cc70..bf0ad43c 100644 --- a/contracts/ts/seats/nodebuildimage/index.js +++ b/contracts/ts/seats/nodebuildimage/index.js @@ -25,6 +25,7 @@ export const events = { stepEnded: "step-ended", toolchainReported: "toolchain-reported", stepProgress: "step-progress", + askTaken: "ask-taken", }; export const family = { name: "node-build-image.events", maxAgeDays: 7 }; @@ -45,14 +46,18 @@ export function toolchainReportedSubject(node) { export const toolchainReportedSubjectAll = "mesh.seat.node-build-image.event.toolchain-reported.*"; -export const stepProgressStream = "SEAT_node-build-image_step-progress"; - export function stepProgressSubject(node) { return "mesh.seat.node-build-image.event.step-progress." + node; } export const stepProgressSubjectAll = "mesh.seat.node-build-image.event.step-progress.*"; +export function askTakenSubject(node) { + return "mesh.seat.node-build-image.event.ask-taken." + node; +} + +export const askTakenSubjectAll = "mesh.seat.node-build-image.event.ask-taken.*"; + export const deliverySubject = "mesh.seat.node-build-image.accept.delivery"; export const checkSubject = "mesh.seat.node-build-image.accept.check"; diff --git a/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index c8907fdb..2addf5c4 100644 --- a/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -15,6 +15,7 @@ export declare const events: { readonly stepEnded: "step-ended"; readonly toolchainReported: "toolchain-reported"; readonly stepProgress: "step-progress"; + readonly askTaken: "ask-taken"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -25,6 +26,16 @@ export interface AskArtifact { "version"?: string; } +/** an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602) */ +export interface AskTaken { + /** the ask's id */ + "ask": string; + "asker": string; + "at": string; + /** the node that took it, as in the subject */ + "node": string; +} + export interface BuildAnswer { "artifacts": BuiltArtifact[]; "imports"?: string[]; @@ -392,23 +403,31 @@ export interface StepAsk { "step"?: StepRun; /** the layout version */ "v"?: number; + /** the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573) */ + "version"?: string; } -/** a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at */ +/** an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548) */ export interface StepEnded { - "artifact": string; + "artifact"?: string; + /** the ask's id, by which a reader takes only its own asks */ + "ask": string; + /** who asked it */ + "asker": string; + /** when it ended (RFC 3339 time) */ + "at": string; /** for a rerun, the caller the runtime proved */ "by"?: string; /** the packages not run because the ask's kept results held them */ "carried"?: KeptResult[]; - "commit": string; + "commit"?: string; "ended"?: string; - "fingerprint": string; + "fingerprint"?: string; /** the ask's id */ "id": string; /** the packages that passed at their digest, for the runner to keep */ "kept"?: KeptResult[]; - "module": string; + "module"?: string; /** the node that ran it, as in the subject */ "node": string; /** never passed for a step that ran nothing */ @@ -417,12 +436,12 @@ export interface StepEnded { "output"?: string; /** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */ "output-object"?: string; - "pipeline": string; + "pipeline"?: string; "rerun"?: boolean; /** why it failed or could not run, naming the part not heard */ "said"?: string; "started"?: string; - "step": "lint" | "unit" | "integration"; + "step"?: "lint" | "unit" | "integration"; "tests"?: TestCounts; /** the layout version */ "v"?: number; @@ -513,6 +532,8 @@ export interface TestCounts { export interface Toolchain { /** when (RFC 3339 time) */ "at"?: string; + /** the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472) */ + "digest"?: string; /** go or typescript */ "language": string; "recipe": string; @@ -643,3 +664,6 @@ export type ToolchainReportedEvent = ToolchainReported; /** is the body of the event step-progress. */ export type StepProgressEvent = StepProgress; +/** is the body of the event ask-taken. */ +export type AskTakenEvent = AskTaken; + diff --git a/contracts/ts/seats/nodebuildimage/parts/build-step.js b/contracts/ts/seats/nodebuildimage/parts/build-step.js index 155a68be..400cdf1c 100644 --- a/contracts/ts/seats/nodebuildimage/parts/build-step.js +++ b/contracts/ts/seats/nodebuildimage/parts/build-step.js @@ -15,6 +15,7 @@ export const events = { stepEnded: "step-ended", toolchainReported: "toolchain-reported", stepProgress: "step-progress", + askTaken: "ask-taken", }; export {}; diff --git a/contracts/ts/seats/nodebuildnode/index.d.ts b/contracts/ts/seats/nodebuildnode/index.d.ts index 69ed8aee..7d0eaaf3 100644 --- a/contracts/ts/seats/nodebuildnode/index.d.ts +++ b/contracts/ts/seats/nodebuildnode/index.d.ts @@ -27,6 +27,7 @@ export declare const events: { readonly stepEnded: "step-ended"; readonly toolchainReported: "toolchain-reported"; readonly stepProgress: "step-progress"; + readonly askTaken: "ask-taken"; }; export declare const family: { readonly name: "node-build-node.events"; readonly maxAgeDays: 7 }; @@ -43,12 +44,14 @@ export declare function toolchainReportedSubject(node: string): string; export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-node.event.toolchain-reported.*"; -export declare const stepProgressStream: "SEAT_node-build-node_step-progress"; - export declare function stepProgressSubject(node: string): string; export declare const stepProgressSubjectAll: "mesh.seat.node-build-node.event.step-progress.*"; +export declare function askTakenSubject(node: string): string; + +export declare const askTakenSubjectAll: "mesh.seat.node-build-node.event.ask-taken.*"; + export declare const deliverySubject: "mesh.seat.node-build-node.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-node.accept.check"; @@ -109,6 +112,8 @@ export declare function detectedStepsByModuleArtifactKey(module: string, artifac export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { module: string; artifact: string } | undefined; export type AskArtifact = buildStep.AskArtifact; +export type AskTaken = buildStep.AskTaken; +export type AskTakenEvent = buildStep.AskTakenEvent; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; diff --git a/contracts/ts/seats/nodebuildnode/index.js b/contracts/ts/seats/nodebuildnode/index.js index 6cda8ea2..90d22dca 100644 --- a/contracts/ts/seats/nodebuildnode/index.js +++ b/contracts/ts/seats/nodebuildnode/index.js @@ -25,6 +25,7 @@ export const events = { stepEnded: "step-ended", toolchainReported: "toolchain-reported", stepProgress: "step-progress", + askTaken: "ask-taken", }; export const family = { name: "node-build-node.events", maxAgeDays: 7 }; @@ -45,14 +46,18 @@ export function toolchainReportedSubject(node) { export const toolchainReportedSubjectAll = "mesh.seat.node-build-node.event.toolchain-reported.*"; -export const stepProgressStream = "SEAT_node-build-node_step-progress"; - export function stepProgressSubject(node) { return "mesh.seat.node-build-node.event.step-progress." + node; } export const stepProgressSubjectAll = "mesh.seat.node-build-node.event.step-progress.*"; +export function askTakenSubject(node) { + return "mesh.seat.node-build-node.event.ask-taken." + node; +} + +export const askTakenSubjectAll = "mesh.seat.node-build-node.event.ask-taken.*"; + export const deliverySubject = "mesh.seat.node-build-node.accept.delivery"; export const checkSubject = "mesh.seat.node-build-node.accept.check"; diff --git a/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index c8907fdb..2addf5c4 100644 --- a/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -15,6 +15,7 @@ export declare const events: { readonly stepEnded: "step-ended"; readonly toolchainReported: "toolchain-reported"; readonly stepProgress: "step-progress"; + readonly askTaken: "ask-taken"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -25,6 +26,16 @@ export interface AskArtifact { "version"?: string; } +/** an ask a build seat's holder took, said when it takes any ask, whatever its asker (design 47 G section 602) */ +export interface AskTaken { + /** the ask's id */ + "ask": string; + "asker": string; + "at": string; + /** the node that took it, as in the subject */ + "node": string; +} + export interface BuildAnswer { "artifacts": BuiltArtifact[]; "imports"?: string[]; @@ -392,23 +403,31 @@ export interface StepAsk { "step"?: StepRun; /** the layout version */ "v"?: number; + /** the version MAJOR.MINOR.PATCH the builder reserved for the build, passed to it as MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G section 573) */ + "version"?: string; } -/** a pipeline step's end, said by the build seat's holder on the node that ran it (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at */ +/** an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G section 602): a pipeline step's end names the pipeline, the commit and the fingerprint it was asked at, which the runner takes only from that holder's account (ADR 0666 sections 7, 21); the end of any other ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (design 47 G section 548) */ export interface StepEnded { - "artifact": string; + "artifact"?: string; + /** the ask's id, by which a reader takes only its own asks */ + "ask": string; + /** who asked it */ + "asker": string; + /** when it ended (RFC 3339 time) */ + "at": string; /** for a rerun, the caller the runtime proved */ "by"?: string; /** the packages not run because the ask's kept results held them */ "carried"?: KeptResult[]; - "commit": string; + "commit"?: string; "ended"?: string; - "fingerprint": string; + "fingerprint"?: string; /** the ask's id */ "id": string; /** the packages that passed at their digest, for the runner to keep */ "kept"?: KeptResult[]; - "module": string; + "module"?: string; /** the node that ran it, as in the subject */ "node": string; /** never passed for a step that ran nothing */ @@ -417,12 +436,12 @@ export interface StepEnded { "output"?: string; /** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */ "output-object"?: string; - "pipeline": string; + "pipeline"?: string; "rerun"?: boolean; /** why it failed or could not run, naming the part not heard */ "said"?: string; "started"?: string; - "step": "lint" | "unit" | "integration"; + "step"?: "lint" | "unit" | "integration"; "tests"?: TestCounts; /** the layout version */ "v"?: number; @@ -513,6 +532,8 @@ export interface TestCounts { export interface Toolchain { /** when (RFC 3339 time) */ "at"?: string; + /** the sha256: digest of the language's toolchain program as the build module runs it: the program of the module that provides that language's toolchain, never the build module's own (design 47 G section 472) */ + "digest"?: string; /** go or typescript */ "language": string; "recipe": string; @@ -643,3 +664,6 @@ export type ToolchainReportedEvent = ToolchainReported; /** is the body of the event step-progress. */ export type StepProgressEvent = StepProgress; +/** is the body of the event ask-taken. */ +export type AskTakenEvent = AskTaken; + diff --git a/contracts/ts/seats/nodebuildnode/parts/build-step.js b/contracts/ts/seats/nodebuildnode/parts/build-step.js index 155a68be..400cdf1c 100644 --- a/contracts/ts/seats/nodebuildnode/parts/build-step.js +++ b/contracts/ts/seats/nodebuildnode/parts/build-step.js @@ -15,6 +15,7 @@ export const events = { stepEnded: "step-ended", toolchainReported: "toolchain-reported", stepProgress: "step-progress", + askTaken: "ask-taken", }; export {}; diff --git a/sdk/buildstep/ask.go b/sdk/buildstep/ask.go index 8a065726..a739fe8a 100644 --- a/sdk/buildstep/ask.go +++ b/sdk/buildstep/ask.go @@ -204,15 +204,18 @@ func AskStepWatch(askID string) (string, error) { // StepAsk is one ask of a build module: the part's `StepAsk`. type StepAsk struct { - V int `json:"v"` - ID string `json:"id"` - Kind string `json:"kind"` - Language string `json:"language"` - Asker string `json:"asker"` - At time.Time `json:"at,omitzero"` - Build *BuildJob `json:"build,omitempty"` - Check *CheckJob `json:"check,omitempty"` - Compose *ComposeJob `json:"compose,omitempty"` + V int `json:"v"` + ID string `json:"id"` + Kind string `json:"kind"` + Language string `json:"language"` + Asker string `json:"asker"` + At time.Time `json:"at,omitzero"` + // Version is the version MAJOR.MINOR.PATCH the builder reserved for the build, which the build module passes as + // MESH_VERSION; empty for a build that reserves none, which stamps none (design 47 G §573). + Version string `json:"version,omitempty"` + Build *BuildJob `json:"build,omitempty"` + Check *CheckJob `json:"check,omitempty"` + Compose *ComposeJob `json:"compose,omitempty"` // Run is a run ask's job (run.go): a program built from a change and run over the heads, on the Go seat's check // stream alone, as a compose. Run *RunJob `json:"run,omitempty"` diff --git a/sdk/buildstep/ask_test.go b/sdk/buildstep/ask_test.go index 1ad37ce0..41a83ca7 100644 --- a/sdk/buildstep/ask_test.go +++ b/sdk/buildstep/ask_test.go @@ -300,9 +300,30 @@ func TestAskSubjectsStreamsAndKeys(t *testing.T) { t.Fatal(err) } goSubj, _ := StepEndedSubject(AskGo, "ace") - if back, err := DecodeStepEnded(goSubj, raw); err != nil || back.Fingerprint != "fp-0123" || back.Tests.Unit != 14 { + if back, err := DecodeStepEnded(goSubj, raw); err != nil || back.Fingerprint != "fp-0123" || back.Tests.Unit != 14 || + back.Asker != stepAsk().Asker || back.Ask != testAskID { t.Errorf("round trip: %+v %v", back, err) } + // 1966/CT12 (G §548, §602): the end of an ask outside a pipeline names its asker, the ask and when, never a commit. + at := time.Date(2026, 11, 11, 10, 3, 0, 0, time.UTC) + build := StepEnded{ID: testAskID, Asker: "mesh-delivery-builder", Ask: testAskID, Node: "ace", Outcome: OutcomePassed, At: at} + if raw, err := EncodeStepEnded(build); err != nil || strings.Contains(string(raw), "commit") { + t.Errorf("a builder's build end: %s %v", raw, err) + } else if back, err := DecodeStepEnded(goSubj, raw); err != nil || back.Asker != "mesh-delivery-builder" || !back.At.Equal(at) { + t.Errorf("a builder's build end reads back as %+v %v", back, err) + } + for name, bad := range map[string]func(*StepEnded){ + "no asker": func(e *StepEnded) { e.Asker = "" }, + "no time": func(e *StepEnded) { e.At = time.Time{} }, + "a commit": func(e *StepEnded) { e.Commit = "0123456789abcdef0123456789abcdef01234567" }, + "a fingerprint": func(e *StepEnded) { e.Fingerprint = "fp-0123" }, + } { + e := build + bad(&e) + if _, err := EncodeStepEnded(e); err == nil { + t.Errorf("an end outside a pipeline with %s is said", name) + } + } g14, _ := StepEndedSubject(AskGo, "g14") if _, err := DecodeStepEnded(g14, raw); err == nil || !strings.Contains(err.Error(), "said on g14's subject") { t.Errorf("an end said on another node's subject: %v", err) diff --git a/sdk/buildstep/pipelinestep.go b/sdk/buildstep/pipelinestep.go index b5157cc0..40e51f74 100644 --- a/sdk/buildstep/pipelinestep.go +++ b/sdk/buildstep/pipelinestep.go @@ -106,16 +106,20 @@ func (r StepRun) check(id string) error { return nil } -// StepEnded is a pipeline step's end, said by the build seat's holder on the node that ran it. +// StepEnded is an ask's end, said by the build seat's holder on the node that ran it, whatever its asker (design 47 G +// §602): a pipeline step's end names its pipeline, step, module, artifact, commit and fingerprint; the end of any other +// ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (G §548). type StepEnded struct { V int `json:"v"` ID string `json:"id"` - Pipeline string `json:"pipeline"` - Step string `json:"step"` - Module string `json:"module"` - Artifact string `json:"artifact"` - Commit string `json:"commit"` - Fingerprint string `json:"fingerprint"` + Asker string `json:"asker"` + Ask string `json:"ask"` + Pipeline string `json:"pipeline,omitempty"` + Step string `json:"step,omitempty"` + Module string `json:"module,omitempty"` + Artifact string `json:"artifact,omitempty"` + Commit string `json:"commit,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` Node string `json:"node"` Outcome string `json:"outcome"` Tests *TestCounts `json:"tests,omitempty"` @@ -128,6 +132,7 @@ type StepEnded struct { By string `json:"by,omitempty"` Started time.Time `json:"started,omitzero"` Ended time.Time `json:"ended,omitzero"` + At time.Time `json:"at,omitzero"` } // EndOf is the end of the step a holds, on node, with outcome: the ask's names copied, so the runner can match the ask, @@ -137,7 +142,7 @@ func EndOf(a StepAsk, node, outcome string) (StepEnded, error) { return StepEnded{}, fmt.Errorf("buildstep: the ask %s is no pipeline step", a.ID) } s := a.Step - e := StepEnded{V: StepEndedVersion, ID: a.ID, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Artifact: s.Artifact, + e := StepEnded{V: StepEndedVersion, ID: a.ID, Asker: a.Asker, Ask: a.ID, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Artifact: s.Artifact, Commit: s.Commit, Fingerprint: s.Fingerprint, Node: node, Outcome: outcome, Rerun: s.Rerun, By: s.By} return e, e.check() } @@ -149,6 +154,20 @@ func (e StepEnded) check() error { if err := checkNode(e.Node); err != nil { return err } + if e.Pipeline == "" { + // G §548, §602: the end of an ask outside a pipeline names its asker, the ask and when, and no commit or + // fingerprint. + if !slices.Contains(StepOutcomes, e.Outcome) { + return fmt.Errorf("buildstep: the end of %s says %q, not one of %s", e.ID, e.Outcome, strings.Join(StepOutcomes, ", ")) + } + if e.Asker == "" || e.Ask == "" || e.At.IsZero() { + return fmt.Errorf("buildstep: the end of %s, an ask outside a pipeline, names no asker, ask or time", e.ID) + } + if e.Commit != "" || e.Fingerprint != "" { + return fmt.Errorf("buildstep: the end of %s names a commit or a fingerprint and no pipeline", e.ID) + } + return nil + } if !slices.Contains(PipelineSteps, e.Step) { return fmt.Errorf("buildstep: the end of %s names the step %q", e.ID, e.Step) } diff --git a/sdk/buildstep/toolchain.go b/sdk/buildstep/toolchain.go index 9b655926..61d13d70 100644 --- a/sdk/buildstep/toolchain.go +++ b/sdk/buildstep/toolchain.go @@ -16,11 +16,14 @@ const ToolchainVersion = 1 // the part build-step's `Toolchain`, the layout every node-build seat gives that state; the three build modules share // this neutral form (a seat's generated type is one package per seat). A test reads each seat's fixture strictly into it. type Toolchain struct { - V int `json:"v,omitempty"` - Language string `json:"language"` - Version string `json:"version"` - Recipe string `json:"recipe"` - At time.Time `json:"at,omitzero"` + V int `json:"v,omitempty"` + Language string `json:"language"` + Version string `json:"version"` + Recipe string `json:"recipe"` + // Digest is the sha256: digest of the language's toolchain program as the build module runs it, the program of the + // module that provides that language's toolchain, never the build module's own (design 47 G §472). + Digest string `json:"digest,omitempty"` + At time.Time `json:"at,omitzero"` } func checkToolchainLanguage(language string) error { diff --git a/sdk/buildstep/toolchain_test.go b/sdk/buildstep/toolchain_test.go index 3cff6eea..01afe441 100644 --- a/sdk/buildstep/toolchain_test.go +++ b/sdk/buildstep/toolchain_test.go @@ -44,8 +44,8 @@ func TestTheToolchainsValueAndKeyAreTheSeatsLayout(t *testing.T) { read++ } } - if read != 3 { - t.Errorf("%d toolchain fixtures read, want 3", read) + if read != 6 { // 1966/CT12 (G §472): each seat's fixture with its program's digest too + t.Errorf("%d toolchain fixtures read, want 6", read) } raw, err := EncodeToolchain(Toolchain{Language: "go", Version: "go1.27.1", Recipe: "3", At: time.Date(2026, 11, 5, 10, 0, 0, 0, time.UTC)}) if err != nil { From 848c2b035e77c7449b061dcc81ca6701651cf8c5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:28:23 +0100 Subject: [PATCH 6/9] BL3: the build seat's ask carries the version the build reserved, empty for a build reserving none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §573, §370 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/build/reserve_test.go | 18 ++++++++++++++++++ .../internal/build/run.go | 9 +++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index d2c85fdf..ae1627f1 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -9,6 +9,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "mesh-delivery-builder/internal/versions" @@ -65,7 +66,16 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { } res.filled = true + // Design 47 G §573: the build's ask carries the reserved version. + var askedVersions []string + r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { + askedVersions = append(askedVersions, a.Version) + return r.answer(a) + } v := r.consider(t, "modules/m") + if len(askedVersions) == 0 || slices.ContainsFunc(askedVersions, func(v string) bool { return v != "0.1.5" }) { + t.Fatalf("the asks carried versions %q, want the reserved 0.1.5 on each", askedVersions) + } if v.State != builds.BuildDone || len(res.reserved) != 1 || !slices.Equal(res.reserved[0], []string{"m:tool"}) || res.series[0] != "0.1" { t.Fatalf("build %+v reserved %v in %v, want done and one reservation of m:tool in 0.1", v.State, res.reserved, res.series) } @@ -81,11 +91,19 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { r2 := newRig(t, goBundle) res2 := &fakeReservations{filled: true} r2.b.Reserved = res2 + var unversioned []string + r2.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { + unversioned = append(unversioned, a.Version) + return r2.answer(a) + } var journal []string r2.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) } if v := r2.consider(t, "modules/m"); v.State != builds.BuildDone || len(res2.reserved) != 0 { t.Errorf("a manifest with no MAJOR.MINOR: state %s, reserved %v", v.State, res2.reserved) } + if len(unversioned) == 0 || slices.ContainsFunc(unversioned, func(v string) bool { return v != "" }) { + t.Errorf("a build reserving none asked with versions %q, want none", unversioned) + } // G §505: the builder writes in its journal, naming the module, that it reserved nothing. if !slices.ContainsFunc(journal, func(l string) bool { return strings.HasPrefix(l, "m: ") && strings.Contains(l, "not MAJOR.MINOR") }) { t.Errorf("journal %q, want a line naming m and saying its version is not MAJOR.MINOR", journal) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e17baad3..e99287a3 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -552,7 +552,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB b.mu.Lock() b.builds[key] = withdraw b.mu.Unlock() - result = b.build(run, module, m, plan, planErr, prior, only, b.notes(cl)) + result = b.build(run, module, m, plan, planErr, prior, only, b.notes(cl), reserved.Version) b.mu.Lock() delete(b.builds, key) b.mu.Unlock() @@ -646,6 +646,7 @@ func failed(format string, args ...any) builds.BuilderBuild { // lane is the build of a plan's tasks of one build module on one node. type lane struct { module, node string // node is the one that pulled the lane's ask + version string // the version the build reserved, MAJOR.MINOR.PATCH, empty for a build that reserves none (design 47 G §573) tasks []Task toolchain *builds.ToolchainUsed digests map[string]string // artifact -> digest @@ -656,7 +657,7 @@ type lane struct { err error } -func (b *Builder) build(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, planErr error, prior *builds.BuilderBuild, only map[string]bool, asked *askNotes) builds.BuilderBuild { +func (b *Builder) build(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, planErr error, prior *builds.BuilderBuild, only map[string]bool, asked *askNotes, version string) builds.BuilderBuild { switch { case planErr != nil: return failed("%s", planErr) @@ -697,7 +698,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module } } // One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3). - lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}}) + lanes = append(lanes, &lane{module: bm, version: version, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}}) } if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { b.handLanes(key, len(lanes)) @@ -952,7 +953,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes if needsFetch(l.tasks) { job.Fetch = &req } - ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: builds.BuilderModule, At: time.Now(), Build: job} + ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: builds.BuilderModule, At: time.Now(), Version: l.version, Build: job} if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 { // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. From bd3727a9000ca43377896bef5e32ecbad260777f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:35:04 +0100 Subject: [PATCH 7/9] BL10: tagged versions and version.numbered go from the builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state tag-asks and its bounds, builder_tag, builder_tag_asks and builder_untag, the checker's tag-wanted consumer, the branch end's removal of tagged versions, the numbering of a check's build under a tag and the event version.numbered (its contract in builds, its emit, its manifest line) are removed; a forge message still wakes a floor. builder_publish_again refuses a held build, since nothing says a version numbered. Builds: design 47 G §373, §321, §349, §413 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-builder/builds/builds.go | 2 +- .../mesh-delivery-builder/builds/numbered.go | 57 ----- .../cmd/mesh-delivery-builder-tools/main.go | 51 +--- .../cmd/mesh-delivery-builder/branchend.go | 163 +------------ .../mesh-delivery-builder/integration_test.go | 128 ++-------- .../cmd/mesh-delivery-builder/main.go | 76 +----- .../cmd/mesh-delivery-builder/refused_test.go | 39 +-- .../cmd/mesh-delivery-builder/tagbounds.go | 132 ---------- .../cmd/mesh-delivery-builder/versions.go | 55 ----- .../internal/build/numbers.go | 35 +-- .../internal/build/run.go | 12 +- .../internal/build/run_test.go | 15 +- .../internal/versions/again.go | 224 +---------------- .../internal/versions/branchend.go | 92 ------- .../internal/versions/claim.go | 20 +- .../internal/versions/heard.go | 19 -- .../internal/versions/store.go | 33 +-- .../internal/versions/tagbound.go | 75 ------ .../internal/versions/tools.go | 13 +- .../internal/versions/versions.go | 41 ---- .../internal/versions/versions_test.go | 225 ++---------------- .../mesh-delivery-builder/manifest_test.go | 10 +- modules/mesh-delivery-builder/module.json | 7 - 23 files changed, 103 insertions(+), 1421 deletions(-) delete mode 100644 modules/mesh-delivery-builder/builds/numbered.go delete mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go delete mode 100644 modules/mesh-delivery-builder/internal/versions/branchend.go delete mode 100644 modules/mesh-delivery-builder/internal/versions/tagbound.go diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index f282bb74..fe8d7f7b 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -1,6 +1,6 @@ // Package builds holds what the builder module owns and no seat carries (novox/hq ADR 0634 §1): the layout of its state // `builds` (a build per module and fingerprint) and the rules its writer keeps, the question a registry seat's holder asks -// it before it stores a publish, its event version.numbered and the entries of its build log. Other modules import it from +// it before it stores a publish and the entries of its build log. Other modules import it from // the builder by its published version (ADR 0573 §1); the builder's manifest declares it as a package artifact. package builds diff --git a/modules/mesh-delivery-builder/builds/numbered.go b/modules/mesh-delivery-builder/builds/numbered.go deleted file mode 100644 index 0eca558a..00000000 --- a/modules/mesh-delivery-builder/builds/numbered.go +++ /dev/null @@ -1,57 +0,0 @@ -package builds - -import ( - "encoding/json" - "fmt" -) - -// The builder's event version.numbered (design 51 *Events of the builder*, *The floor*): said once per version the builder -// numbers, on its own family `version` (seven days), apart from its builds' ends, so a waiter on a build's end does not -// wake on it. A build module consumes it to publish a build's version with the held digest, which stores nothing. -const ( - // VersionNumberedEvent is the event's local name. - VersionNumberedEvent = "version.numbered" - // VersionNumberedSubject is its subject, as a consumer's filter names it. - VersionNumberedSubject = "mesh.mod." + BuilderModule + ".event." + VersionNumberedEvent -) - -// VersionNumbered is the body of VersionNumberedEvent: a numbered version of a name, the build it was numbered for, and -// what the build module needs to publish it. -type VersionNumbered struct { - // Name is the numbered name, as builder_versions lists it; Kind its artifact's kind. - Name string `json:"name"` - Kind string `json:"kind"` - Version string `json:"version"` - Tag string `json:"tag,omitempty"` - // Module and Fingerprint are the build the version was numbered for; Commit its source commit. - Module string `json:"module"` - Fingerprint string `json:"fingerprint"` - Commit string `json:"commit,omitempty"` - // Digest is the built artifact's, when the build held one; Seat the registry seat it is published through. - Digest string `json:"digest,omitempty"` - Seat string `json:"seat,omitempty"` - // Head is the head the version answers, when it answers a tag ask. - Head string `json:"head,omitempty"` - // Again is when builder_publish_again said it once more, so the bus's dedup does not drop it. - Again string `json:"again,omitempty"` -} - -// EncodeVersionNumbered writes the body, refusing one that names no name, version, module or fingerprint. -func EncodeVersionNumbered(n VersionNumbered) ([]byte, error) { - if n.Name == "" || n.Version == "" || n.Module == "" || n.Fingerprint == "" { - return nil, fmt.Errorf("builds: %s names its name, version, module and fingerprint", VersionNumberedEvent) - } - return json.Marshal(n) -} - -// DecodeVersionNumbered reads the body, with the same refusal. -func DecodeVersionNumbered(data []byte) (VersionNumbered, error) { - var n VersionNumbered - if err := json.Unmarshal(data, &n); err != nil { - return VersionNumbered{}, fmt.Errorf("builds: %s is not readable: %w", VersionNumberedEvent, err) - } - if _, err := EncodeVersionNumbered(n); err != nil { - return VersionNumbered{}, err - } - return n, nil -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 1411f36c..bbd75eaf 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -35,9 +35,8 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error) // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for // the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer. -// tags writes a tag-asks value and waits for the builder's answer; ta reads tag-asks. func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader, - tags queueAsker, ta, fw, of versions.Reader) []stdio.Tool { + fw, of versions.Reader) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } // handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error. handAct := func(key string, value []byte, err error) (any, error) { @@ -57,32 +56,10 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q } return map[string]any{"key": key, "name": a.Name, "do": a.Do, "version": a.Version, "answer": a.Answer}, nil } - tagAct := func(key string, value []byte, err error) (any, error) { - if err != nil { - return nil, err - } - raw, err := tags(key, value) - if err != nil { - return nil, err - } - return versions.DecodeTagAsk(raw) - } nameFP := map[string]any{"type": "object", "required": []string{"name", "fp", "why"}, "properties": map[string]any{ "name": map[string]any{"type": "string", "description": "the numbered name, /, as builder_versions lists it"}, "fp": map[string]any{"type": "string", "description": "the fingerprint, src2:"}, "why": map[string]any{"type": "string", "description": "why, in plain words"}}} - branch := func(head bool) map[string]any { - req := []string{"repository", "branch", "why"} - props := map[string]any{ - "repository": map[string]any{"type": "string", "description": "/"}, - "branch": map[string]any{"type": "string", "description": "the branch; its cleaned name is the tag"}, - "why": map[string]any{"type": "string", "description": "why, in plain words"}} - if head { - req = append(req, "head") - props["head"] = map[string]any{"type": "string", "description": "the commit the tagged version is built from"} - } - return map[string]any{"type": "object", "required": req, "properties": props} - } return []stdio.Tool{ {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, Input: nameFP, Run: func(args map[string]any) (any, error) { @@ -95,18 +72,6 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q Run: func(args map[string]any) (any, error) { return handAct(build.NewFloorAgain(str(args, "name"), str(args, "why"), stdio.CallerOf(args), now())) }}, - {Name: versions.TagTool, Description: versions.TagDescription, Input: branch(true), - Run: func(args map[string]any) (any, error) { - return tagAct(versions.NewTagAsk(str(args, "repository"), str(args, "branch"), str(args, "head"), str(args, "why"), stdio.CallerOf(args), now())) - }}, - {Name: versions.UntagTool, Description: versions.UntagDescription, Input: branch(false), - Run: func(args map[string]any) (any, error) { - return tagAct(versions.NewUntag(str(args, "repository"), str(args, "branch"), str(args, "why"), stdio.CallerOf(args), now())) - }}, - {Name: versions.TagAsksTool, Description: versions.TagAsksDescription, - Input: map[string]any{"type": "object", "properties": map[string]any{ - "repository": map[string]any{"type": "string", "description": "only this repository's asks, /"}}}, - Run: func(args map[string]any) (any, error) { return versions.AnswerTagAsks(ta, str(args, "repository")) }}, {Name: versions.OldFormTool, Description: versions.OldFormDescription, Input: map[string]any{"type": "object", "properties": map[string]any{ "drop": map[string]any{"type": "string", "description": "a retired node to drop from the count by hand"}, @@ -373,18 +338,6 @@ func main() { } return e.Value, true, nil }} - ts := stdio.State(versions.TagAsksState) - tags := askKept(ts, versions.TagAsksState, versions.TagAsksBucket, "the ask", 25*time.Second, func(v []byte) bool { - a, err := versions.DecodeTagAsk(v) - return err == nil && !a.Answered.IsZero() - }) - tr := versions.Reader{Keys: ts.Keys, Get: func(key string) (json.RawMessage, bool, error) { - e, err := ts.Get(key) - if err != nil || e == nil { - return nil, false, err - } - return e.Value, true, nil - }} fs := stdio.State(versions.FloorWaitsState) fr := versions.Reader{Keys: fs.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := fs.Get(key) @@ -417,7 +370,7 @@ func main() { } return e.Value, true, nil }} - tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, tags, tr, fr, ofr) + tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) // The builder's expiring keys (state bounds): what each bounds and when it ends. tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go index 4e2e1cd9..5a1cfcf5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/branchend.go @@ -2,173 +2,26 @@ package main import ( "context" - "encoding/json" - "errors" - "strings" - "time" "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - - "mesh-delivery-builder/internal/build" - "mesh-delivery-builder/internal/versions" ) -// forgeStream is the forge bridge's stream `forge` as the bus names it, where the builder's durable consumer hears a -// branch's end (design 51 *Tagged versions*). +// forgeStream is the forge bridge's stream `forge` as the bus names it, where the builder's durable consumer hears the +// forge's messages: each wakes a floor whose bridge_tags was silent (design 51 *The floor*). No tagged version is +// removed on a branch's end, none being made (design 47 G §373). const forgeStream = "FORGE" -// forgeBranchDeleted is the forge bridge's kind for a branch's end (forgeBranchDeleted in the group's mesh-sdk). -const forgeBranchDeleted = "branch-deleted" - -// checkerAsksStream is the checker's family `asks` as the bus names it, and tagWantedSubject its event tag-wanted -// (design 51 *Tagged versions*, DLV-C3): on the checker's own stream, so its subject is mesh.stream..event.*. -const ( - checkerAsksStream = "ASKS" - tagWantedSubject = "mesh.stream." + checkerModule + ".event.tag-wanted" -) - -// forgeEnd is the part of a forge message the builder reads. -type forgeEnd struct { - Kind string `json:"kind"` - Repository string `json:"repository"` - Branch string `json:"branch"` - Action string `json:"action"` +// forgeWakes hears the forge bridge's stream: every message wakes a floor whose bridge_tags was silent, and is acknowledged. +type forgeWakes struct { + wake func(ctx context.Context) + logf func(string, ...any) } -// ended says whether the message ends a branch: merged, a pull closed, or branch-deleted. -func (f forgeEnd) ended() bool { - switch f.Kind { - case forgemsg.ForgeMerged, forgeBranchDeleted: - return f.Branch != "" - case forgemsg.ForgePull: - return f.Action == forgemsg.PullClosed && f.Branch != "" - } - return false -} - -// branchEnds hears the forge bridge's stream: every message wakes a floor whose bridge_tags was silent, and a branch's -// end removes its tagged versions, acknowledged only once all are removed. -type branchEnds struct { - acts versions.Acts - findings *findings - tagAsks func() jetstream.KeyValue - wake func(ctx context.Context) - logf func(string, ...any) - now func() time.Time -} - -func (h *branchEnds) heard(ctx context.Context, m jetstream.Msg) { +func (h *forgeWakes) heard(ctx context.Context, m jetstream.Msg) { if h.wake != nil { h.wake(ctx) } - var f forgeEnd - if err := json.Unmarshal(m.Data(), &f); err != nil || !f.ended() { - h.ack(m) - return - } - end, err := h.acts.BranchEnded(ctx, f.Repository, f.Branch) - if err != nil { - h.logf("the end of %s %s: the versions cannot be read, heard again: %v", f.Repository, f.Branch, err) - h.again(m) - return - } - id := f.Repository + "/" + f.Branch - if end.Done { - h.findings.clear(ctx, versions.RemovalNotDone, meshconditionsclient.ScopeModule, id) - if kv := h.tagAsks(); kv != nil { - if err := kv.Delete(ctx, versions.TagAskKey(f.Repository, f.Branch)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - h.logf("the tag ask of %s %s stays in %s: %v", f.Repository, f.Branch, versions.TagAsksState, err) - } - } - h.ack(m) - return - } - since := h.now() - if md, err := m.Metadata(); err == nil { - since = md.Timestamp - } - if h.now().Sub(since) >= versions.RemovalFindingAfter { - h.findings.raise(ctx, finding{kind: versions.RemovalNotDone, scope: meshconditionsclient.ScopeModule, id: id, - summary: end.Finding(f.Repository, f.Branch, since), - headline: "A branch's tagged versions are not removed", - explanation: "The branch ended, and the registry seats did not say removed for every version tagged for it within 15 minutes.", - resolved: "Every seat said removed.", - needs: "Look at the seats named: " + strings.Join(end.Seats(), ", ")}) - } - h.logf("the end of %s %s: remove asked of %d tagged versions, heard again in a minute", f.Repository, f.Branch, len(end.Left)) - h.again(m) -} - -func (h *branchEnds) ack(m jetstream.Msg) { if err := m.Ack(); err != nil { h.logf("acknowledging %s: %v", m.Subject(), err) } } - -// again leaves the message for the bus to give again in a minute: the retry is the bus's redelivery, not a timer here. -func (h *branchEnds) again(m jetstream.Msg) { - if err := m.NakWithDelay(versions.RemoveAgainAfter); err != nil { - h.logf("leaving %s for the bus to give again: %v", m.Subject(), err) - } -} - -// tagWanted takes the checker's tag-wanted in: written to tag-asks as builder_tag writes it, answered by its watch. -func tagWanted(ctx context.Context, kv jetstream.KeyValue, m jetstream.Msg, by string, now time.Time, logf func(string, ...any)) { - if m.Subject() != tagWantedSubject { - _ = m.Ack() // the family's other events (the verdicts) are not the builder's - return - } - var t versions.TagWanted - err := json.Unmarshal(m.Data(), &t) - var key string - var value []byte - if err == nil { - key, value, err = versions.TagWantedAsk(t, by, now) - } - if err != nil { - logf("%s holds what is no tag-wanted, left: %v", m.Subject(), err) - _ = m.Ack() - return - } - if kv == nil { - logf("tag-wanted of %s %s is heard again: %s is not on the bus yet", t.Repository, t.Branch, versions.TagAsksState) - _ = m.NakWithDelay(versions.RemoveAgainAfter) - return - } - if _, err := kv.Put(ctx, key, value); err != nil { - logf("tag-wanted of %s %s is heard again: %v", t.Repository, t.Branch, err) - _ = m.Nak() - return - } - _ = m.Ack() -} - -// tagsVia is build.Builder.Tags over the state tag-asks: the asks of the repository, read when a check's build is -// looked at; none while the state is not bound. -func tagsVia(kv func() jetstream.KeyValue, logf func(string, ...any)) build.TagOf { - return func(ctx context.Context, repository, head string) (string, bool) { - store := kv() - if store == nil { - return "", false - } - lister, err := store.ListKeysFiltered(ctx, versions.EscapeToken(repository)+".>") - if err != nil { - logf("the tag asks of %s cannot be read, so its check builds no tagged version: %v", repository, err) - return "", false - } - var asks []versions.TagAsk - for key := range lister.Keys() { - e, err := store.Get(ctx, key) - if err != nil { - continue - } - if t, err := versions.DecodeTagAsk(e.Value()); err == nil { - asks = append(asks, t) - } - } - return versions.TagFor(asks, repository, head) - } -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 6d270415..c277bc6c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -29,13 +29,11 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/keys" @@ -217,8 +215,6 @@ type bed struct { builds jetstream.KeyValue tools map[string]jetstream.KeyValue instances jetstream.KeyValue - tagAsks jetstream.KeyValue - tagBounds jetstream.KeyValue packages jetstream.KeyValue // the indexer's packages, served by its read tool and views family out *saidLog // the builder's process log, every instance's stop func() // stops the builder started last @@ -306,12 +302,6 @@ func startBedWith(t *testing.T, closed bool) *bed { viewstest.Serve(t, ctx, js, ctl, assignerModule, "assignments", assignerModule+".assigner_read_assignments", readers...) b.builds = mk(builds.BuilderBuildsBucket, builds.BuilderBuildsHistory) b.instances = mk(instancesBucket, 1) - // The tag asks and their bounds (design 51 *Tagged versions*): the bounds declared key-ttl, so an ended bound leaves the - // bus's marker a watch hears. - b.tagAsks = mk(versions.TagAsksBucket, 1) - if b.tagBounds, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: versions.TagAskBoundsBucket, History: 1, LimitMarkerTTL: time.Minute}); err != nil { - t.Fatal(err) - } b.out = &saidLog{} // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog) @@ -525,16 +515,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 first := "src2:" + strings.Repeat("1", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, first, 1) digest := b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, fp, 2) - // DLV-B6: a tag ask kept and never numbered (no check builds its head) is narrated once its bound ends on the bus, with - // what it waits on, and its bound is written again; no timer runs in the builder. - key, ask, err := versions.NewTagAsk("novox/mesh-sdk", "feat/x", "abc", "a group member's check", "me@shanks", time.Now()) - if err != nil { - t.Fatal(err) - } - if _, err := b.tagAsks.Put(b.ctx, key, ask); err != nil { - t.Fatal(err) - } - b.start(t, "n1", map[string]string{"MESH_BUILDER_TAG_ASK_BOUND": "1s", "MESH_BUILDER_STEP_BOUND": "1s"}) + b.start(t, "n1", map[string]string{"MESH_BUILDER_STEP_BOUND": "1s"}) v := b.await(t, "netcheck", isState(builds.BuildDone)) for _, a := range []string{"tools-go", "tools-typescript"} { es := reg.of(t, "netcheck/"+a) @@ -577,39 +558,17 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) } } - said := b.awaitSaid(t, "the tag ask of novox/mesh-sdk feat/x at abc (feat-x) is not answered after") - if !strings.Contains(said, "the check's build of abc is not claimed yet") { - t.Errorf("the narration = %q, want what the ask waits on", said) - } - // And in the build log, under the builder's own key (the ruling of 13:17: build.said). - if !slices.ContainsFunc(b.logged(t, builds.BuilderModule), func(e builds.BuildLogEntry) bool { - return e.Event == builds.BuildSaidEvent && e.Why == versions.SaidUnanswered && e.FP == key && e.Commit == "abc" && - strings.Contains(e.Line, "the check's build of abc is not claimed yet") - }) { - t.Errorf("the build log of %s = %+v, want the unanswered tag ask said", builds.BuilderModule, b.logged(t, builds.BuilderModule)) - } - if e, err := b.tagAsks.Get(b.ctx, key); err != nil { - t.Fatal(err) - } else if a, _ := versions.DecodeTagAsk(e.Value()); a.Answered.IsZero() || !a.Unanswered() { - t.Errorf("the tag ask = %+v, want kept and answered, no version yet", a) - } - for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(50 * time.Millisecond) { - if _, err := b.tagBounds.Get(b.ctx, key); err == nil { - break - } else if time.Now().After(deadline) { - t.Fatalf("the bound is not written again after its narration: %v", err) - } - } // Design 32, ADR 0615 §1: a package's name the indexer writes later reaches the builder by its packages' views family // (never their bucket), and looks at the module again; the done bundles are kept, not numbered twice. if _, err := b.packages.Put(b.ctx, "netcheck.0.1", []byte(`{"v":1,"name":"netcheck","package":"netcheck/lib","series":"0.1"}`)); err != nil { t.Fatal(err) } b.awaitSaid(t, "names a package of netcheck") - if got := reg.numberedOf("netcheck/tools-go"); len(got) != 1 || got[0] != "0.1.5" { - t.Errorf("version.numbered of netcheck/tools-go = %v, want 0.1.5 once", got) + // Design 47 G §349: version.numbered is no longer said, for a number or for a late name. + if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { + t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got) } - // DLV-B10: builder_publish_again after a published takes a new number, and says it numbered. + // DLV-B10: builder_publish_again of a published, held build is refused: nothing says a version numbered (§349). for _, a := range []string{"tools-go", "tools-typescript"} { reg.published(t, "netcheck/"+a, "0.1.5", digestOf(a)) } @@ -618,8 +577,14 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if err != nil { t.Fatal(err) } - if a := reg.answered(t, key, again); a.Refused || a.Do != versions.AgainRenumber || a.Version != "0.1.6" { - t.Errorf("builder_publish_again = %+v, want a new number, 0.1.6", a) + if a := reg.answered(t, key, again); !a.Refused { + t.Errorf("builder_publish_again = %+v, want a refusal", a) + } + // A numbered version a holder will refuse below, written to the state as the builder numbers one. + refusable, _ := json.Marshal(versions.Entry{Name: "netcheck/tools-go", Kind: versions.KindBundle, Series: "0.1", Version: "0.1.6", Module: "netcheck", + Fingerprint: "src2:" + strings.Repeat("6", 64), Seat: ociartifactregistry.Seat, State: versions.Numbered, At: time.Now().UTC()}) + if _, err := reg.versions.Put(b.ctx, versions.Key("netcheck/tools-go", "0.1.6"), refusable); err != nil { + t.Fatal(err) } // Running-testing note (review round 1): two claims never get one number on the bus's own compare-and-set, as // the builder's state versions keeps them: eight claims of one name at once, eight versions. @@ -644,8 +609,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if len(got) != 8 { t.Errorf("eight claims of one name at once were numbered %v, want eight versions", slices.Sorted(maps.Keys(got))) } - if got := reg.numberedOf("netcheck/tools-go"); len(got) != 2 || got[1] != "0.1.6" { - t.Errorf("version.numbered of netcheck/tools-go = %v, want 0.1.6 said after 0.1.5", got) + if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { + t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got) } // DLV-B10: a build module's bound running out fails the trunk build, which is the builder's finding build-failed; the // fingerprint built again keeps its number, said with why, and its done build clears the finding. @@ -683,33 +648,6 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 return e.Version == "0.1.6" && e.State == versions.Refused && e.Reason == "the cap" && e.RefusedDigest == digestOf("other") }) }) - // DLV-B6: a branch's end (the forge bridge's merged) asks the seat to remove each version tagged for it, the reason - // tag-gone, and the seat's removed turns it removed. - tagged := versions.Entry{Name: "mesh-sdk/go", Kind: versions.KindPackage, Series: "0.1", Version: "0.1.15-feat-x.1", Tag: "feat-x", Module: "mesh-sdk", - Fingerprint: "src2:" + strings.Repeat("5", 64), Repository: "novox/mesh-sdk", Seat: gopackageregistry.Seat, State: versions.Published, At: time.Now().UTC()} - raw, _ := json.Marshal(tagged) - if _, err := reg.versions.Put(b.ctx, versions.Key(tagged.Name, tagged.Version), raw); err != nil { - t.Fatal(err) - } - if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(forgeBridgeModule, "forge"), - []byte(`{"kind":"`+forgemsg.ForgeMerged+`","repository":"novox/mesh-sdk","branch":"feat/x"}`)); err != nil { - t.Fatal(err) - } - for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) { - reg.mu.Lock() - asked := slices.ContainsFunc(reg.removes, func(a map[string]any) bool { - return a["name"] == tagged.Name && a["version"] == tagged.Version && a["reason"] == "tag-gone" - }) - reg.mu.Unlock() - if asked { - break - } - if time.Now().After(deadline) { - t.Fatalf("the seat's remove of %s@%s was not asked at the branch's end: %v", tagged.Name, tagged.Version, reg.removes) - } - } - reg.outcome(t, gopackageregistry.SubjectRemoved, versions.Outcome{Name: tagged.Name, Version: tagged.Version, Kind: versions.KindPackage, Holder: "gitea", Reason: "tag-gone"}) - reg.awaitOf(t, tagged.Name, func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Removed }) } // TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no @@ -1180,8 +1118,7 @@ type registry struct { b *bed versions, findings, reasks jetstream.KeyValue mu sync.Mutex - numbered []builds.VersionNumbered - removes []map[string]any + numbered []string // the bodies said on the subject version.numbered was said on before design 47 G §349 } func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *registry { @@ -1209,27 +1146,12 @@ func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *regis t.Fatal(err) } } - // The Go seat's remove, as its holder answers it; each call is kept. - rm, err := b.js.Conn().Subscribe("mesh.seat."+gopackageregistry.Seat+".call.remove.>", func(m *nats.Msg) { - var args map[string]any - _ = json.Unmarshal(m.Data, &args) - r.mu.Lock() - r.removes = append(r.removes, args) - r.mu.Unlock() - _ = m.Respond([]byte(`{"result":{},"node":"ctl"}`)) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = rm.Unsubscribe() }) nc := b.js.Conn() - // As a build module hears it: on the contract's subject, read by the contract's decoder (BG2's late publish). - sub, err := nc.Subscribe(builds.VersionNumberedSubject, func(m *nats.Msg) { - if nb, err := builds.DecodeVersionNumbered(m.Data); err == nil { - r.mu.Lock() - r.numbered = append(r.numbered, nb) - r.mu.Unlock() - } + // Nothing says version.numbered any more (§349): its old subject is watched, to say so if something does. + sub, err := nc.Subscribe("mesh.mod."+builds.BuilderModule+".event.version.numbered", func(m *nats.Msg) { + r.mu.Lock() + r.numbered = append(r.numbered, string(m.Data)) + r.mu.Unlock() }) if err != nil { t.Fatal(err) @@ -1335,14 +1257,14 @@ func (r *registry) awaitFinding(t *testing.T, key string, raised bool) { t.Fatalf("the finding %s raised = %v within 20 s, want %v", key, !raised, raised) } -// numberedOf are the version.numbered the builder said for name, in order. +// numberedOf are the bodies said as version.numbered for name: none since design 47 G §349. func (r *registry) numberedOf(name string) []string { r.mu.Lock() defer r.mu.Unlock() var out []string - for _, nb := range r.numbered { - if nb.Name == name { - out = append(out, nb.Version) + for _, body := range r.numbered { + if strings.Contains(body, `"`+name+`"`) { + out = append(out, body) } } return out diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 659e7caa..c04e2884 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -117,13 +117,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err) } } - // An unanswered tag ask is narrated every TagAskBound (design 51: five minutes); a test gives a shorter one. - tagAskBound := versions.TagAskBound - if v := strings.TrimSpace(getenv("MESH_BUILDER_TAG_ASK_BOUND")); v != "" { - if tagAskBound, err = time.ParseDuration(v); err != nil { - return fmt.Errorf("MESH_BUILDER_TAG_ASK_BOUND: %w", err) - } - } tools := &buildrun.ToolchainSet{} // Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot // (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity. @@ -152,7 +145,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs // (the seat verb held, novox/hq issue 1937) before it leaves the key done, on its calls connection. // The builder numbers each trunk build at its claim (novox/hq design 51 *The versions*): its state versions, bound once - // the bus holds it; the floor's three sources asked as its own account; each new number said as version.numbered. + // the bus holds it; the floor's three sources asked as its own account. versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf) found := &findings{kv: findingsKV, logf: logger.Printf, now: time.Now} floorWaits := &versions.FloorWaits{Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))} @@ -164,12 +157,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) // A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32). numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, - Names: versions.NamesIn(indexerPackages{packages}), - Said: func(nb builds.VersionNumbered) { - if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil { - logger.Printf("%v", err) - } - }} + Names: versions.NamesIn(indexerPackages{packages})} // A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder // answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read. pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf) @@ -193,15 +181,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput := &unputs{bounds: pubBoundsKV.KV, store: b.Claimer.Store, looking: b.Looking, findings: found, bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf} b.Left, b.Ended = unput.left, unput.ended - tagAsksKV := process.AwaitBucket(ctx, js, conn, versions.TagAsksBucket, "builder_tag and builder_untag are not answered", logger.Printf) - // A check's build of a head a kept tag ask names numbers its packages tagged (design 51 *Tagged versions*), and the ask - // is marked numbered with its versions. - b.Tags = tagsVia(tagAsksKV.KV, logger.Printf) - b.Numbers = build.VersionNumbers{N: numberer, TagNumbered: tagNumbered(tagAsksKV.KV, logger.Printf, time.Now)} - // An ask not answered within five minutes is narrated with what it waits on: its bound's end on the bus says when. - boundsKV := process.AwaitBucket(ctx, js, conn, versions.TagAskBoundsBucket, "an unanswered tag ask is not narrated", logger.Printf) - bounds := &tagBounds{asks: tagAsksKV.KV, bounds: boundsKV.KV, bound: tagAskBound, waits: floorWaits.Waiting, logf: logger.Printf, now: time.Now, - log: func(e builds.BuildLogEntry) { b.Log(e) }} + b.Numbers = build.VersionNumbers{N: numberer} // The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled // and version-released heard; nothing is reserved before the fill. reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf) @@ -210,7 +190,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. - acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Done: doneVia(b), Reconsider: b.Reconsider, Remove: removeVia(calls.Call), + acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Done: doneVia(b), Reconsider: b.Reconsider, BuildAgain: func(ctx context.Context, module, fp, why, by string) (string, bool) { r := b.BuildAgain(ctx, module, fp, why, by, buildAgainBound) if r.Refused != "" { @@ -393,23 +373,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, rw, eachReAsk) }() } }) - // A tag ask or a branch's removal by hand (builder_tag, builder_untag) is written to tag-asks and answered there - // (design 51 *Tagged versions*); the checker's tag-wanted and the branch's end join it on their consumers (DLV-B6). - tagAsksKV.Ready(func(kv jetstream.KeyValue) { - each := func(e jetstream.KeyValueEntry) { - tagAsked(ctx, acts, kv, e, logger.Printf) - if !opOf(e) { - bounds.disarm(ctx, e.Key()) - } else if t, err := versions.DecodeTagAsk(e.Value()); err == nil { - bounds.asked(ctx, e.Key(), t) - } - } - if tw, err := settle(ctx, kv, each); err != nil { - process.NotWatched(logger.Printf, versions.TagAsksBucket, err, "builder_tag and builder_untag are not answered until the builder starts again") - } else { - go func() { errs <- follow(ctx, tw, each) }() - } - }) pubBoundsKV.Ready(func(kv jetstream.KeyValue) { each := func(e jetstream.KeyValueEntry) { if reserved.ended(ctx, e) { @@ -435,14 +398,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } watchBounds(ctx, kv, each, found.notHeard(ctx), errs, logger.Printf) }) - boundsKV.Ready(func(kv jetstream.KeyValue) { - each := func(e jetstream.KeyValueEntry) { bounds.ended(ctx, e) } - if bw, err := settle(ctx, kv, each); err != nil { - process.NotWatched(logger.Printf, versions.TagAskBoundsBucket, err, "an unanswered tag ask is not narrated until the builder starts again") - } else { - go func() { errs <- follow(ctx, bw, each) }() - } - }) // A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the // builder alone may read the stream, so it answers there. process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { @@ -453,24 +408,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, qw, eachRead) }() } }) - // A branch's end removes its tagged versions (design 51 *Tagged versions*, DLV-B6): the forge bridge's merged, a pull's - // closed or branch-deleted on the builder's durable consumer of its stream; each forge message also wakes a floor whose - // bridge_tags was silent. The checker's tag-wanted is kept in tag-asks. A consumer not there yet is said. - ends := &branchEnds{acts: acts, findings: found, tagAsks: tagAsksKV.KV, logf: logger.Printf, now: time.Now, - wake: func(ctx context.Context) { - (&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "") - }} - for _, c := range []struct { - stream, what string - hear func(jetstream.Msg) - }{ - {forgeStream, "a branch's end", func(m jetstream.Msg) { ends.heard(ctx, m) }}, - {checkerAsksStream, "the checker's tag-wanted", func(m jetstream.Msg) { - tagWanted(ctx, tagAsksKV.KV(), m, checkerModule, time.Now(), logger.Printf) - }}, - } { - hear(c.stream, c.what+" is not heard until the builder starts again", c.hear) - } + // Each forge message wakes a floor whose bridge_tags was silent (design 51 *The floor*). A consumer not there yet is said. + ends := &forgeWakes{logf: logger.Printf, wake: func(ctx context.Context) { + (&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "") + }} + hear(forgeStream, "the forge's messages are not heard until the builder starts again", func(m jetstream.Msg) { ends.heard(ctx, m) }) if stopped != nil { return stopped } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index 78fbdfb7..d3d4bf22 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -29,7 +29,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -314,46 +313,16 @@ func (noVersions) Update(context.Context, string, []byte, uint64) (uint64, error func (noVersions) KeysOf(context.Context, string) ([]string, error) { return nil, nil } func aBranchsEndIsHeard(t *testing.T) { - for _, c := range []struct { - f forgeEnd - want bool - }{ - {forgeEnd{Kind: forgemsg.ForgeMerged, Repository: "o/r", Branch: "feat/x"}, true}, - {forgeEnd{Kind: forgeBranchDeleted, Repository: "o/r", Branch: "feat/x"}, true}, - {forgeEnd{Kind: forgemsg.ForgePull, Repository: "o/r", Branch: "feat/x", Action: forgemsg.PullClosed}, true}, - {forgeEnd{Kind: forgemsg.ForgePull, Repository: "o/r", Branch: "feat/x", Action: forgemsg.PullOpened}, false}, - {forgeEnd{Kind: forgemsg.ForgeMerged, Repository: "o/r"}, false}, - } { - if got := c.f.ended(); got != c.want { - t.Errorf("%+v ended = %v, want %v", c.f, got, c.want) - } - } + // A forge message wakes a floor and is acknowledged; no tagged version is removed on a branch's end (design 47 G §373). var logs bytes.Buffer woken := 0 - h := &branchEnds{acts: versions.Acts{Numberer: &versions.Numberer{Versions: versions.Versions{Store: noVersions{}}}}, findings: &findings{logf: t.Logf}, - tagAsks: func() jetstream.KeyValue { return nil }, wake: func(context.Context) { woken++ }, logf: log.New(&logs, "", 0).Printf, now: time.Now} + h := &forgeWakes{wake: func(context.Context) { woken++ }, logf: log.New(&logs, "", 0).Printf} open := &wakeMsg{subject: "mesh.mod.mesh-delivery-forge-bridge.event.forge.o.r.pull", data: []byte(`{"kind":"pull","repository":"o/r","branch":"feat/x","action":"opened"}`)} h.heard(context.Background(), open) merged := &wakeMsg{subject: "mesh.mod.mesh-delivery-forge-bridge.event.forge.o.r.merged", data: []byte(`{"kind":"merged","repository":"o/r","branch":"feat/x"}`)} h.heard(context.Background(), merged) - if !open.acked || !merged.acked || woken != 2 { - t.Errorf("an open pull acked %v, a merge with nothing tagged acked %v, floors woken %d times", open.acked, merged.acked, woken) - } - // The checker's family holds its verdicts beside tag-wanted: a verdict is acknowledged and not kept; a tag-wanted with - // tag-asks not bound is heard again. - verdict := &wakeMsg{subject: "mesh.stream.mesh-delivery-checker.event.verdict", data: []byte(`{"repository":"o/r"}`)} - wanted := &wakeMsg{subject: tagWantedSubject, data: []byte(`{"repository":"o/r","branch":"feat/x","head":"abc","tag":"feat-x"}`)} - for _, m := range []*wakeMsg{verdict, wanted} { - tagWanted(context.Background(), nil, m, checkerModule, time.Now(), t.Logf) - } - if !verdict.acked || verdict.delayed != 0 || wanted.acked || wanted.delayed != versions.RemoveAgainAfter || !strings.HasPrefix(tagWantedSubject, "mesh.stream.") { - t.Errorf("a verdict acked %v (delayed %v), a tag-wanted acked %v (delayed %v)", verdict.acked, verdict.delayed, wanted.acked, wanted.delayed) - } - unreadable := &wakeMsg{subject: merged.subject, data: merged.data} - h.acts.Numberer.Versions.Store = failingVersions{} - h.heard(context.Background(), unreadable) - if unreadable.acked || unreadable.delayed != versions.RemoveAgainAfter { - t.Errorf("a branch's end the versions cannot answer: acked %v, given again after %v", unreadable.acked, unreadable.delayed) + if !open.acked || !merged.acked || woken != 2 || open.delayed != 0 || merged.delayed != 0 { + t.Errorf("an open pull acked %v, a merge acked %v, floors woken %d times", open.acked, merged.acked, woken) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go deleted file mode 100644 index c9dc9d67..00000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go +++ /dev/null @@ -1,132 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - - "mesh-delivery-builder/internal/versions" -) - -// tagBounds narrates a tag ask the builder has not answered within five minutes, with what it waits on (design 51 -// *Tagged versions*): each unanswered ask has a key in the state tag-ask-bounds written with its own time to live, and -// the bus's marker of its end, heard on the watch, is what narrates it and writes the bound again. No timer. -type tagBounds struct { - asks func() jetstream.KeyValue - bounds func() jetstream.KeyValue - waits func() []versions.FloorWait - logf func(string, ...any) - // log writes a narration into the builder's build log (build.said); nil writes none. - log func(builds.BuildLogEntry) - now func() time.Time - bound time.Duration -} - -// arm writes key's bound when none stands: a Create, so a standing bound keeps its time and a restart arms each -// unanswered ask whose bound ended while the builder was away. -func (tb *tagBounds) arm(ctx context.Context, key string) { - kv := tb.bounds() - if kv == nil { - return - } - if _, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(tb.bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { - tb.logf("the bound of the tag ask %s is not written, so it is not narrated past five minutes: %v", key, err) - } -} - -// disarm removes key's bound: the ask was answered or ended. -func (tb *tagBounds) disarm(ctx context.Context, key string) { - if kv := tb.bounds(); kv != nil { - _ = kv.Delete(ctx, key) - } -} - -// asked is told each value of tag-asks: an unanswered ask is armed, any other disarmed. -func (tb *tagBounds) asked(ctx context.Context, key string, t versions.TagAsk) { - if t.Unanswered() { - tb.arm(ctx, key) - return - } - tb.disarm(ctx, key) -} - -// ended is told each change of tag-ask-bounds: a bound whose time ended (the bus's marker, a purge on the watch) on an -// ask still unanswered narrates it and arms it again. -func (tb *tagBounds) ended(ctx context.Context, e jetstream.KeyValueEntry) { - if e.Operation() != jetstream.KeyValuePurge { - return - } - kv := tb.asks() - if kv == nil { - return - } - got, err := kv.Get(ctx, e.Key()) - if err != nil { - return // the ask ended with its branch - } - t, err := versions.DecodeTagAsk(got.Value()) - if err != nil || !t.Unanswered() { - return - } - var waits []versions.FloorWait - if tb.waits != nil { - waits = tb.waits() - } - line := versions.NarrateUnanswered(t, waits, tb.now()) - tb.logf("%s", line) - // In the build log too (design 51 *Tagged versions*): under the builder's own key, no build being claimed yet, the - // ask's key standing for the fingerprint. - if tb.log != nil { - tb.log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: builds.BuilderModule, FP: e.Key(), Commit: t.Head, - At: tb.now().UTC(), Line: line, Why: versions.SaidUnanswered}) - } - tb.arm(ctx, e.Key()) -} - -// tagNumbered marks the tag ask of a check's build's head numbered with its versions in tag-asks, by compare-and-set. -func tagNumbered(kv func() jetstream.KeyValue, logf func(string, ...any), now func() time.Time) func(context.Context, versions.BuildClaim, versions.Numbering) { - return func(ctx context.Context, c versions.BuildClaim, got versions.Numbering) { - store := kv() - if store == nil { - return - } - lister, err := store.ListKeysFiltered(ctx, versions.EscapeToken(c.Repository)+".>") - if err != nil { - logf("the tag ask of %s at %s is not marked numbered: %v", c.Repository, c.Head, err) - return - } - for key := range lister.Keys() { - for range 3 { - e, err := store.Get(ctx, key) - if err != nil { - break - } - t, err := versions.DecodeTagAsk(e.Value()) - if err != nil { - break - } - t, changed := t.WithNumbered(c.Module, c.Head, got.Versions, got.Names, now()) - if !changed { - break - } - data, err := json.Marshal(t) - if err != nil { - break - } - if _, err = store.Update(ctx, key, data, e.Revision()); err == nil { - logf("%s: tag ask of %s %s: %s", key, t.Repository, t.Branch, t.Answer) - break - } - if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { - logf("the tag ask %s is not marked numbered: %v", key, err) - break - } - } - } - } -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index 8fa2946d..fbe610ac 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/process" @@ -208,19 +207,6 @@ func floorVia(call toolCaller) versions.FloorSources { } } -// emitNumbered says the builder's event version.numbered (design 51 *Events of the builder*), on its own family apart -// from its builds' ends; the name and version are the bus's dedup id. -func emitNumbered(ctx context.Context, js jetstream.JetStream, cred process.Account, b builds.VersionNumbered) error { - id := "numbered-" + versions.Key(b.Name, b.Version) - if b.Again != "" { - id += "-again-" + b.Again - } - if err := emitEvent(ctx, js, cred, builds.VersionNumberedEvent, id, b); err != nil { - return fmt.Errorf("saying %s for %s@%s: %w", builds.VersionNumberedEvent, b.Name, b.Version, err) - } - return nil -} - // emitEvent says one of the builder's events, mesh.mod..event., with the SDK's envelope headers and id as // the bus's dedup id. func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, event, id string, body any) error { @@ -320,44 +306,3 @@ func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (boo return cur.State == builds.BuildDone, nil } } - -// removeVia calls a registry seat's remove for one tagged version, as the builder's own account (design 51 *Tagged -// versions*): the reason tag-gone; the seat says removed on its events, or answers kept when another tag names the -// digest (the registry part's RemoveAnswer: kept, reason, tags), which ends the remove. -func removeVia(call toolCaller) func(ctx context.Context, seat string, e versions.Entry) (versions.RemoveOutcome, error) { - return func(_ context.Context, seat string, e versions.Entry) (versions.RemoveOutcome, error) { - raw, err := call("seat:"+seat+".remove", map[string]any{"name": e.Name, "version": e.Version, "tag": e.Tag, "reason": "tag-gone"}) - if err != nil { - return versions.RemoveOutcome{}, err - } - var a ociartifactregistry.RemoveAnswer // the four registry seats share the part's answer - if len(raw) > 0 && json.Unmarshal(raw, &a) != nil { - return versions.RemoveOutcome{}, fmt.Errorf("%s's remove answered what is no answer of it: %s", seat, raw) - } - return versions.RemoveOutcome{Kept: a.Kept, Reason: a.Reason, Tags: a.Tags}, nil - } -} - -// tagAsked answers an unanswered value of tag-asks by compare-and-set at its revision, said in the log. -func tagAsked(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { - if e.Operation() != jetstream.KeyValuePut { - return - } - t, err := versions.DecodeTagAsk(e.Value()) - if err != nil { - logf("%s.%s holds what is not a tag ask, left: %v", versions.TagAsksState, e.Key(), err) - return - } - if !t.Answered.IsZero() { - return - } - t = acts.TagAsked(ctx, t, time.Now()) - logf("%s: %s of %s %s by %s (%s): %s", e.Key(), map[bool]string{true: "untag", false: "tag ask"}[t.Untag], t.Repository, t.Branch, t.By, t.Why, t.Answer) - data, err := json.Marshal(t) - if err != nil { - return - } - if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { - logf("%s.%s: the answer could not be written: %v", versions.TagAsksState, e.Key(), err) - } -} diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 279a8345..26dd0fe5 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -18,18 +18,14 @@ import ( // Numbers numbers a trunk build at its claim (novox/hq design 51 *The versions*, ADR 0606, ADR 0617 §1) and records the // digests it was built with. Nil numbers nothing. type Numbers interface { - // tag is a check's build's tag (TagOf), empty for a trunk build; done the digests of its key when it is done already - // (versions.BuildClaim.Done), nil otherwise. - Number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, tag string, done map[string]string) (versions.Numbering, error) + // done is the digests of the build's key when it is done already (versions.BuildClaim.Done), nil otherwise. + Number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, done map[string]string) (versions.Numbering, error) Built(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, got versions.Numbering, made []builds.BuiltArtifact) error } // VersionNumbers is Numbers over the builder's state versions. type VersionNumbers struct { N *versions.Numberer - // TagNumbered is told the versions a check's build of a tag ask's head was numbered (the ask is then marked numbered - // in tag-asks, design 51 *Tagged versions*); nil tells nobody. - TagNumbered func(ctx context.Context, c versions.BuildClaim, got versions.Numbering) } // ClaimOf is the claim a module value's trunk build is numbered by: a package per source revision (Src), the rest per @@ -46,18 +42,10 @@ func ClaimOf(module string, m meshcatalog.ModulesByModuleValue, plan Plan) versi return c } -func (v VersionNumbers) Number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, tag string, done map[string]string) (versions.Numbering, error) { +func (v VersionNumbers) Number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, done map[string]string) (versions.Numbering, error) { c := ClaimOf(module, m, plan) c.Done = done - if tag != "" { - // A check's build of a head a tag ask names: its packages tagged, below the next version (design 51 *Tagged versions*). - c.Tag, c.Head, c.Revision, c.Merge = tag, m.Commit, "", 0 - } - got, err := v.N.Number(ctx, c) - if err == nil && tag != "" && len(got.Versions) > 0 && v.TagNumbered != nil { - v.TagNumbered(ctx, c, got) - } - return got, err + return v.N.Number(ctx, c) } func (v VersionNumbers) Built(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, got versions.Numbering, made []builds.BuiltArtifact) error { @@ -68,22 +56,15 @@ func (v VersionNumbers) Built(ctx context.Context, module string, m meshcatalog. return v.N.Built(ctx, ClaimOf(module, m, plan), got, digests) } -// TagOf answers the cleaned tag a kept tag ask names for a repository's head (the builder's state tag-asks, design 51 -// *Tagged versions*): a check's build of that head numbers its packages tagged at its claim. -type TagOf func(ctx context.Context, repository, head string) (tag string, ok bool) - -// number numbers a build before its claim, a trunk build's artifacts or a check's build's packages under its tag: false +// number numbers a trunk build's artifacts before its claim: false // when it waits for its floor (a package, design 51 *The floor*), and then nothing is claimed; the floor's wake looks at // the module again. -func (b *Builder) number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, tag string) (versions.Numbering, bool) { +func (b *Builder) number(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan) (versions.Numbering, bool) { if b.Numbers == nil { return versions.Numbering{}, true } - var done map[string]string - if tag == "" { - done = b.doneDigests(ctx, module, m.Fp2) - } - got, err := b.Numbers.Number(ctx, module, m, plan, tag, done) + done := b.doneDigests(ctx, module, m.Fp2) + got, err := b.Numbers.Number(ctx, module, m, plan, done) for _, s := range got.Said { b.logf("%s at %s: %s", module, m.Fp2, s.Line) // And in the build log (design 51 *The versions*: it says a version was kept, and why), between the build's start and end. diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e99287a3..cd045595 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -97,8 +97,6 @@ type Builder struct { Numbers Numbers // Reserved reserves the version of a trunk build at its start (design 47 G §369); nil reserves nothing. Reserved Reservations - // Tags answers a check's build's tag from the kept tag asks; nil numbers no check's build. - Tags TagOf // Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine // answered when last asked, and counted, every node the assignments names, present or away, a node with no statement // stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in @@ -485,17 +483,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB byHand := b.hands[key] != nil b.mu.Unlock() var numbering versions.Numbering - tag := "" - if !trunk && b.Tags != nil && b.refOf(module, m.Fp2) != "" { - tag, _ = b.Tags(ctx, repotrunk.RepositoryName(m.Repository), m.Commit) - } - if (trunk || tag != "") && planErr == nil && !nothing { + if trunk && planErr == nil && !nothing { var goOn bool - if numbering, goOn = b.number(ctx, module, m, plan, tag); !goOn { + if numbering, goOn = b.number(ctx, module, m, plan); !goOn { return } } - reserving := trunk && tag == "" && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) + reserving := trunk && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) if reserving && b.waitFill(module, m) { return } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 0339f41e..4d28e0a9 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -434,18 +434,13 @@ func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) r := newRig(t, imageArt) r.puller = "ace" trunkModule(t, r) - // DLV-B6: a check's build of a head a kept tag ask names is numbered under that tag at its claim. + // Design 47 G §373: a candidate's build is numbered under no tag, so it numbers nothing. nums := &fakeNumbers{} - var tagAsked []string r.b.Numbers = nums - r.b.Tags = func(_ context.Context, repository, head string) (string, bool) { - tagAsked = append(tagAsked, repository+"@"+head) - return "feat-x", head == headX - } r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() - if !slices.Equal(nums.tags, []string{"feat-x"}) || len(tagAsked) != 1 || !strings.HasSuffix(tagAsked[0], "@"+headX) { - t.Errorf("the check's build numbered with tags %v (asked %v), want feat-x for its head", nums.tags, tagAsked) + if nums.numbered != 0 { + t.Errorf("a candidate's build was numbered %d times, want none", nums.numbered) } v := valueOf(t, r.st, r.key) if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { @@ -514,13 +509,11 @@ type fakeNumbers struct { versions map[string]string numbered int built map[string]string - tags []string done map[string]string // the done key's digests the last look numbered with refused string // answered as Numbering.Refused } -func (f *fakeNumbers) Number(_ context.Context, _ string, _ meshcatalog.ModulesByModuleValue, _ Plan, tag string, done map[string]string) (versions.Numbering, error) { - f.tags = append(f.tags, tag) +func (f *fakeNumbers) Number(_ context.Context, _ string, _ meshcatalog.ModulesByModuleValue, _ Plan, done map[string]string) (versions.Numbering, error) { f.done = done f.numbered++ return versions.Numbering{Wait: f.wait, Versions: f.versions, Refused: f.refused}, nil diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 2f3e814d..52a3decf 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -2,19 +2,15 @@ package versions import ( "context" - "encoding/json" "errors" "fmt" - "sort" "strings" "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // What builder_publish_again does with one fingerprint of a name (design 51 *Tools of the builder*), after a holder failed. const ( - // AgainPublish says version.numbered again for the recorded version: the build module publishes the held digest. + // AgainPublish is a held build whose recorded version is published again: refused since design 47 G §349 (no version.numbered is said). AgainPublish = "publish" // AgainBuild builds the fingerprint again under the same number: the build's file is gone and no published was said // for the number, so the new digest replaces the recorded one (Numberer.Built). @@ -51,141 +47,15 @@ func PublishAgain(entries []Entry, fingerprint string, built bool) (Again, error case !built: return Again{Do: AgainBuild, Entry: e, Why: fmt.Sprintf("the build's file is gone and no published was said for %s@%s: built again under the same number", e.Name, e.Version)}, nil } - return Again{Do: AgainPublish, Entry: e, Why: fmt.Sprintf("%s@%s (%s) is said numbered again, so its build module publishes the held digest", e.Name, e.Version, e.State)}, nil + return Again{Do: AgainPublish, Entry: e, Why: fmt.Sprintf("%s@%s (%s) is held", e.Name, e.Version, e.State)}, nil } return Again{}, fmt.Errorf("fingerprint %s is not numbered for this name: builder_versions lists what is", fingerprint) } -// The builder's state of tag asks (design 51 *Tagged versions*): one key per repository and branch, removed when the -// branch is merged or deleted or its pull request closed. -const ( - TagAsksState = "tag-asks" - TagAsksBucket = "mesh-delivery-builder_tag-asks" -) - -// The states of a tag ask. -const ( - TagAsked = "asked" - TagNumbered = "numbered" - TagRemoving = "removing" - TagRemoved = "removed" -) - -// TagAsk is one value of tag-asks: an ask of a tagged version for a branch's head, from the checker's tag-wanted or by -// hand (builder_tag), or by hand the removal of the branch's tagged versions (builder_untag, Untag). -type TagAsk struct { - V int `json:"v"` - Repository string `json:"repository"` - Branch string `json:"branch"` - Head string `json:"head,omitempty"` - Tag string `json:"tag"` - By string `json:"by,omitempty"` - Why string `json:"why,omitempty"` - At time.Time `json:"at"` - Untag bool `json:"untag,omitempty"` - // State, Versions (name to version), Left (what a removal still waits on, name@version to its seat) and Answer are - // the builder's. - State string `json:"state,omitempty"` - Versions map[string]string `json:"versions,omitempty"` - Left map[string]string `json:"left,omitempty"` - Answer string `json:"answer,omitempty"` - Answered time.Time `json:"answered,omitzero"` -} - -// TagAskKey is the key of a repository's branch in tag-asks. -func TagAskKey(repository, branch string) string { - return EscapeToken(repository) + "." + EscapeToken(branch) -} - -// NewTagAsk is the value builder_tag writes: its key and the value. The tag is the branch's, cleaned (CleanTag). -func NewTagAsk(repository, branch, head, why, by string, at time.Time) (string, []byte, error) { - return newTagAsk(repository, branch, head, why, by, at, false) -} - -// NewUntag is the value builder_untag writes: its key and the value. -func NewUntag(repository, branch, why, by string, at time.Time) (string, []byte, error) { - return newTagAsk(repository, branch, "", why, by, at, true) -} - -func newTagAsk(repository, branch, head, why, by string, at time.Time, untag bool) (string, []byte, error) { - repository, branch, head, why = strings.TrimSpace(repository), strings.TrimSpace(branch), strings.TrimSpace(head), strings.TrimSpace(why) - switch { - case repository == "" || !strings.Contains(repository, "/"): - return "", nil, errors.New("repository is required: /") - case branch == "": - return "", nil, errors.New("branch is required") - case !untag && head == "": - return "", nil, errors.New("head is required: the commit the tagged version is built from") - case why == "": - return "", nil, errors.New("say why (why): a hand act is recorded") - } - data, err := json.Marshal(TagAsk{V: 1, Repository: repository, Branch: branch, Head: head, Tag: CleanTag(branch), By: by, Why: why, - At: at.UTC(), Untag: untag, State: TagAsked}) - return TagAskKey(repository, branch), data, err -} - -// DecodeTagAsk reads a tag-asks value. -func DecodeTagAsk(data []byte) (TagAsk, error) { - var a TagAsk - if err := json.Unmarshal(data, &a); err != nil { - return a, err - } - if a.V != 1 { - return a, fmt.Errorf("a tag-asks value of version %d, not 1", a.V) - } - return a, nil -} - -// TagAsksAnswer is what builder_tag_asks answers: the open asks, oldest first. -type TagAsksAnswer struct { - Asks []TagAsk `json:"asks"` -} - -// AnswerTagAsks is builder_tag_asks over the state read, only repository's when it is given. -func AnswerTagAsks(r Reader, repository string) (TagAsksAnswer, error) { - keys, err := r.Keys() - if err != nil { - return TagAsksAnswer{}, fmt.Errorf("%s cannot be listed: %w", TagAsksState, err) - } - out := TagAsksAnswer{Asks: []TagAsk{}} - for _, k := range keys { - raw, ok, err := r.Get(k) - if err != nil { - return TagAsksAnswer{}, fmt.Errorf("%s.%s: %w", TagAsksState, k, err) - } - if !ok { - continue - } - a, err := DecodeTagAsk(raw) - if err != nil { - return TagAsksAnswer{}, fmt.Errorf("%s.%s holds what is not a tag ask: %w", TagAsksState, k, err) - } - if repository == "" || a.Repository == repository { - out.Asks = append(out.Asks, a) - } - } - sort.Slice(out.Asks, func(i, j int) bool { return out.Asks[i].At.Before(out.Asks[j].At) }) - return out, nil -} - -// Untagged are repository's tagged versions of tag that are not removed (a group's branch spans repositories, each -// ending apart): what a branch's end, or builder_untag, removes through -// each one's seat (design 51 *Tagged versions*). A version with no tag is never among them. -func Untagged(entries []Entry, repository, tag string) []Entry { - tag = CleanTag(tag) - var out []Entry - for _, e := range entries { - if e.Tag != "" && e.Repository == repository && CleanTag(e.Tag) == tag && e.State != Removed && e.State != Kept { - out = append(out, e) - } - } - return InOrder(out) -} - // The tools' descriptions. const ( PublishAgainDescription = "Publish one fingerprint's version again after its holder failed (design 51): with the build's file held, the " + - "builder says version.numbered again and the build module publishes the held digest; with the file gone, it builds the fingerprint " + + "nothing says a version numbered since design 47 G §349, so a held build is refused; with the file gone, it builds the fingerprint " + "again under the same number, the new digest replacing the recorded one, since no published was said for it; after a published (or " + "a removal) it takes a new number. It refuses a fingerprint not numbered and a version the holder refused for other bytes. A hand " + "act, recorded with its caller and why in the builder's log." @@ -193,15 +63,6 @@ const ( "bridge_tags and the indexer's indexer_requirements, as the builder asks them by itself when a silent source says something. " + "Answers which sources were silent and whether the floor is now known. A name the builder's versions already hold needs no floor: " + "refused, saying so. A hand act." - TagDescription = "Ask a tagged version for a branch's head by hand, as the checker's tag-wanted asks it (design 51 *Tagged versions*): " + - "the builder keeps the ask in its state tag-asks until the branch is merged or deleted, or its pull request closed, and numbers " + - "each package the head builds below the next trunk version (0.1.15-.2). A hand act." - TagAsksDescription = "The open tag asks of the builder's state tag-asks, oldest first: repository, branch, head, tag, who asked and why, " + - "its state (asked, numbered, removing, removed), the versions numbered for it and, while its tagged versions are removed, the " + - "seats that have not yet said removed. With repository, that repository's only. Read-only." - UntagDescription = "Remove a branch's tagged versions by hand, as its merge, its pull request's close or its deletion does (design 51 " + - "*Tagged versions*): the builder calls each seat's remove with the reason tag-gone and marks each version removed once its seat says " + - "removed. Answers the versions it removes and the seats still waited on. A version with no tag is never removed. A hand act." ) // Acts are the builder's hand acts on its versions, answered by its program (the tools bundle writes them to reasks). @@ -213,66 +74,12 @@ type Acts struct { BuildAgain func(ctx context.Context, module, fp, why, by string) (answer string, refused bool) // Reconsider looks at every module again, so a claim asks a name's floor. Reconsider func(ctx context.Context) - // Remove calls a seat's verb remove for one tagged version, reason tag-gone: what it answered, kept or asked. - Remove func(ctx context.Context, seat string, e Entry) (RemoveOutcome, error) // DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not. DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error) // DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not. DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error) } -// TagAsked answers one unanswered value of tag-asks. An untag calls remove on each of the branch's tagged versions' -// seats and leaves each in Left until its seat says removed (Heard, on the seat's events); a call that fails stays in -// Left with its error, asked again by the branch's end. A tag ask is kept: its versions are numbered when the head's -// packages are claimed for the group's check. -func (a Acts) TagAsked(ctx context.Context, t TagAsk, now time.Time) TagAsk { - t.Answered = now.UTC() - if !t.Untag { - t.State = TagAsked - t.Answer = "kept: each package the head builds is numbered " + "-" + t.Tag + ". when its build is claimed for the check" - return t - } - all, err := a.Numberer.Versions.All(ctx) - if err != nil { - t.Answer = "nothing removed: " + err.Error() - return t - } - t.Left, t.Versions = map[string]string{}, map[string]string{} - for _, e := range Untagged(all, t.Repository, t.Tag) { - t.Versions[e.Name] = e.Version - at := e.Name + "@" + e.Version - t.Left[at] = e.Seat - if left, ok := a.removeOne(ctx, e); ok { - delete(t.Left, at) - } else if left != "" { - t.Left[at] = left - } - } - if len(t.Left) == 0 { - t.State, t.Left, t.Answer = TagRemoved, nil, "the branch holds no tagged version: nothing to remove" - return t - } - t.State = TagRemoving - t.Answer = fmt.Sprintf("remove asked of %d tagged versions; each is marked removed when its seat says removed", len(t.Left)) - return t -} - -// removeOne asks e's seat to remove it: done when the seat kept it whole (recorded Kept, no retry); otherwise left, with -// the error of a call that failed. -func (a Acts) removeOne(ctx context.Context, e Entry) (left string, done bool) { - got, err := a.Remove(ctx, e.Seat, e) - if err != nil { - return e.Seat + ": " + err.Error(), false - } - if !got.Kept { - return "", false - } - if err := a.Numberer.keep(ctx, e, got.Reason, got.Tags); err != nil { - return e.Seat + ": kept, and not recorded so: " + err.Error(), false - } - return "", true -} - // PublishAgain is builder_publish_again of name's fingerprint: what it did, the version, and its answer; an error refuses. func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at time.Time) (Again, string, error) { module, _, _ := strings.Cut(name, "/") @@ -288,6 +95,9 @@ func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at tim if err != nil { return Again{}, "", err } + if ag.Do == AgainPublish || (ag.Do == AgainRenumber && done) { + return Again{}, "", errors.New("nothing says a version numbered since design 47 G §349: the version is published under its reservation (§369)") + } e := ag.Entry if ag.Do == AgainRenumber { _, ne, err := a.Numberer.Versions.Number(ctx, Claim{Name: e.Name, Kind: e.Kind, Series: e.Series, Module: e.Module, Fingerprint: e.Fingerprint, @@ -305,14 +115,6 @@ func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at tim } answer += "; " + said } - if ag.Do == AgainPublish || (ag.Do == AgainRenumber && done) { - if a.Numberer.Said != nil { - b := NumberedOf(ag.Entry, "") - b.Again = at.UTC().Format(time.RFC3339Nano) - a.Numberer.Said(b) - } - answer += "; " + builds.VersionNumberedEvent + " said for " + ag.Entry.Version - } return ag, answer, nil } @@ -337,17 +139,3 @@ func (a Acts) FloorAgain(ctx context.Context, name string) (string, error) { a.Reconsider(ctx) return answer + "; builder_versions shows its version once numbered", nil } - -// TagFor is the cleaned tag a kept tag ask names for repository's head: the ask a check's build of that head is -// numbered under (design 51 *Tagged versions*). An untag, or an ask whose branch ended, names none. -func TagFor(asks []TagAsk, repository, head string) (string, bool) { - for _, t := range asks { - if t.Untag || t.State == TagRemoving || t.State == TagRemoved || t.Head == "" { - continue - } - if t.Repository == repository && t.Head == head { - return t.Tag, t.Tag != "" - } - } - return "", false -} diff --git a/modules/mesh-delivery-builder/internal/versions/branchend.go b/modules/mesh-delivery-builder/internal/versions/branchend.go deleted file mode 100644 index 1ced0765..00000000 --- a/modules/mesh-delivery-builder/internal/versions/branchend.go +++ /dev/null @@ -1,92 +0,0 @@ -package versions - -import ( - "context" - "fmt" - "sort" - "strings" - "time" -) - -// A branch's end removes its tagged versions (design 51 *Tagged versions*, DLV-B6): on the forge bridge's `merged`, a -// pull's `closed` or `branch-deleted`, heard on the builder's durable consumer of the stream `forge`, the builder calls -// each seat's `remove` and acknowledges the message only once every seat said `removed` (Heard marks the entry); until -// then the bus gives it again each minute (RemoveAgainAfter) and, past RemovalFindingAfter, the builder's finding names -// the seats still owing. -const ( - RemoveAgainAfter = time.Minute - RemovalFindingAfter = 15 * time.Minute - // RemovalNotDone is the finding's kind. - RemovalNotDone = "tag-removal-not-done" -) - -// BranchEnd is what one hearing of a branch's end did: the versions still not removed (name@version to its seat, with -// the error of a remove that failed), and whether all are gone so the message is acknowledged. -type BranchEnd struct { - Left map[string]string - Done bool -} - -// Seats are the seats still owing a removal, sorted, each once. -func (b BranchEnd) Seats() []string { - seen := map[string]bool{} - var out []string - for _, s := range b.Left { - seat, _, _ := strings.Cut(s, ":") - if !seen[seat] { - seen[seat] = true - out = append(out, seat) - } - } - sort.Strings(out) - return out -} - -// Finding is the removal finding's words: what is left, since when. -func (b BranchEnd) Finding(repository, branch string, since time.Time) string { - var at []string - for v, s := range b.Left { - at = append(at, v+" on "+s) - } - sort.Strings(at) - return fmt.Sprintf("the tagged versions of %s %s are not all removed since its end at %s: %s; the builder asks remove again each minute", - repository, branch, since.UTC().Format(time.RFC3339), strings.Join(at, ", ")) -} - -// BranchEnded removes repository's tagged versions of branch: every one not yet removed is asked of its seat again. -// A store that cannot be read is the error: nothing is acknowledged then. -func (a Acts) BranchEnded(ctx context.Context, repository, branch string) (BranchEnd, error) { - all, err := a.Numberer.Versions.All(ctx) - if err != nil { - return BranchEnd{}, err - } - left := map[string]string{} - for _, e := range Untagged(all, repository, branch) { - at := e.Name + "@" + e.Version - left[at] = e.Seat - if a.Remove == nil { - continue - } - // A seat that kept it whole (another tag names its digest) ends it here: recorded kept, never asked again. - if l, done := a.removeOne(ctx, e); done { - delete(left, at) - } else if l != "" { - left[at] = l - } - } - return BranchEnd{Left: left, Done: len(left) == 0}, nil -} - -// TagWanted is the body of the checker's `tag-wanted` (design 51 *Tagged versions*): a group member's head to number -// tagged versions for. -type TagWanted struct { - Repository string `json:"repository"` - Branch string `json:"branch"` - Head string `json:"head"` - Tag string `json:"tag,omitempty"` -} - -// TagWantedAsk is the tag-asks value a tag-wanted becomes, as builder_tag writes it by hand: by the checker. -func TagWantedAsk(t TagWanted, by string, at time.Time) (string, []byte, error) { - return NewTagAsk(t.Repository, t.Branch, t.Head, "the checker's tag-wanted for a group member's check", by, at) -} diff --git a/modules/mesh-delivery-builder/internal/versions/claim.go b/modules/mesh-delivery-builder/internal/versions/claim.go index 912b7a2c..063e8abe 100644 --- a/modules/mesh-delivery-builder/internal/versions/claim.go +++ b/modules/mesh-delivery-builder/internal/versions/claim.go @@ -6,8 +6,6 @@ import ( "sort" "strings" "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // The registry seats a version is published to (design 51 *The seats*), by what is published. @@ -69,9 +67,6 @@ type BuildClaim struct { Revision string Merge uint64 Artifacts []ClaimArtifact - // Tag numbers a check's build of a head a tag ask names (design 51 *Tagged versions*): its packages only, each a - // tagged version kept with Repository for its branch's end; Head is the head, said in numbered. Empty for a trunk build. - Tag, Head string // Done are the digests of the build's done key, by artifact, when the claim is a look at a build already done: what it // left unnumbered (a package whose name was not known yet, a floor that was silent) is numbered with them, so // version.numbered carries the digest its build module publishes late. Nil for a build still to make. @@ -114,7 +109,6 @@ const ( SaidKept = "kept" SaidWaiting = "waiting" SaidUnnumbered = "unnumbered" - SaidUnanswered = "tag ask unanswered" SaidOldFormClosed = "old-form-closed" ) @@ -133,16 +127,13 @@ func narrationOf(name string, d Decision) Narration { } // Numberer numbers trunk builds at their claim (design 51 *The versions*, DLV-B2, B3): it asks a name's floor only -// when versions holds nothing for it, records a silent floor as a wait (FloorWaits, woken by its source's event), and -// says each new number (Said, the event version.numbered). +// when versions holds nothing for it, and records a silent floor as a wait (FloorWaits, woken by its source's event). type Numberer struct { Versions Versions Floor FloorSources Waits *FloorWaits // Holder is the module holding a seat, whose health condition's clear wakes a silent seat source; nil names none. Holder func(seat string) string - // Said is told each version newly numbered; nil tells nobody. - Said func(builds.VersionNumbered) // Names answers a package's name in its ecosystem; nil numbers a package as NameOf names it. Names PackageNames // Publishing is told each version Built recorded that is still numbered, its holder's word not heard yet: the @@ -159,7 +150,7 @@ func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) sort.Slice(arts, func(i, j int) bool { return arts[i].Artifact < arts[j].Artifact }) for _, a := range arts { seat, kind, ok := SeatOf(a.Kind, a.Language) - if !ok || (c.Tag != "" && kind != KindPackage) { + if !ok { continue } name := NameOf(c.Module, a.Artifact) @@ -194,8 +185,8 @@ func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) if len(entries) == 0 { floor, why = n.askFloor(ctx, seat, name, c) } - d, e, err := n.Versions.Number(ctx, Claim{Name: name, Kind: kind, Series: c.Series, Module: c.Module, Fingerprint: c.Fingerprint, - Commit: c.Commit, Revision: c.Revision, Digest: c.Done[a.Artifact], Seat: seat, Merge: c.Merge, Floor: floor, Tag: c.Tag, Repository: c.Repository}) + d, _, err := n.Versions.Number(ctx, Claim{Name: name, Kind: kind, Series: c.Series, Module: c.Module, Fingerprint: c.Fingerprint, + Commit: c.Commit, Revision: c.Revision, Digest: c.Done[a.Artifact], Seat: seat, Merge: c.Merge, Floor: floor}) if err != nil { return out, err } @@ -214,9 +205,6 @@ func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) n.Waits.Answered(name) } out.Versions[a.Artifact] = d.Version - if !d.Keep && n.Said != nil { - n.Said(NumberedOf(e, c.Head)) - } } return out, nil } diff --git a/modules/mesh-delivery-builder/internal/versions/heard.go b/modules/mesh-delivery-builder/internal/versions/heard.go index 573f4047..6352849d 100644 --- a/modules/mesh-delivery-builder/internal/versions/heard.go +++ b/modules/mesh-delivery-builder/internal/versions/heard.go @@ -27,25 +27,6 @@ type Outcome struct { Tags []string `json:"tags,omitempty"` } -// RemoveOutcome is what a seat's remove answered: kept whole (another tag names its digest), with why and those tags; or -// asked, the seat saying removed on its events. -type RemoveOutcome struct { - Kept bool - Reason string - Tags []string -} - -// keep marks a tagged version kept by its seat, so it is asked no more. -func (n *Numberer) keep(ctx context.Context, e Entry, reason string, tags []string) error { - if reason == "" { - reason = "shared-digest" - } - _, err := n.Versions.Mark(ctx, e.Name, e.Version, func(x *Entry) { - x.State, x.Reason, x.KeptWith = Kept, reason, append([]string(nil), tags...) - }) - return err -} - // Heard takes one seat event in: a numbered version's entry turns published (with its digest when it held none), // refused with the holder's reason, removed with its reason and the event's id, or kept (another tag names its digest) // with the other tags; a by-digest publish (no version) diff --git a/modules/mesh-delivery-builder/internal/versions/store.go b/modules/mesh-delivery-builder/internal/versions/store.go index 4f064420..efe216af 100644 --- a/modules/mesh-delivery-builder/internal/versions/store.go +++ b/modules/mesh-delivery-builder/internal/versions/store.go @@ -55,12 +55,8 @@ func (v Versions) Of(ctx context.Context, name string) ([]Entry, error) { // Claim is one numbering asked of the builder. type Claim struct { Name, Kind, Series, Module, Fingerprint, Commit, Revision, Digest, Seat string - // Repository is the module's repository, kept on a tagged version for its branch's end. - Repository string - // Tag asks a tagged version for this cleaned tag; empty for a trunk version. - Tag string - Merge uint64 - Floor Floor + Merge uint64 + Floor Floor // Fresh numbers anew a fingerprint or revision already numbered: builder_publish_again after a published (design 51 // *Tools of the builder*). Fresh bool @@ -78,23 +74,7 @@ func (v Versions) Number(ctx context.Context, c Claim) (Decision, Entry, error) if err != nil { return Decision{}, Entry{}, err } - var d Decision - if c.Tag != "" { - if !c.Floor.Known() && len(entries) == 0 { - return Decision{Wait: true, Why: "the tagged version waits for its floor"}, Entry{}, nil - } - ver := Tagged(entries, c.Series, c.Tag, c.Floor) - d = Decision{Version: ver, Publish: true, Why: "tagged " + ver} - for _, e := range entries { - // The same fingerprint claimed again for its tag (a check's build looked at again) keeps its number. - if e.Tag == CleanTag(c.Tag) && e.Fingerprint == c.Fingerprint && e.State != Removed { - d = Decision{Version: e.Version, Keep: true, Publish: true, Why: "tagged " + e.Version + ", kept for this fingerprint"} - break - } - } - } else { - d = Decide(fresh(entries, c), c.Kind, c.Series, c.Fingerprint, c.Revision, c.Floor) - } + d := Decide(fresh(entries, c), c.Kind, c.Series, c.Fingerprint, c.Revision, c.Floor) if d.Version == "" { return d, Entry{}, nil } @@ -105,12 +85,9 @@ func (v Versions) Number(ctx context.Context, c Claim) (Decision, Entry, error) } } } - e := Entry{Name: c.Name, Kind: c.Kind, Series: c.Series, Version: d.Version, Tag: c.Tag, Module: c.Module, + e := Entry{Name: c.Name, Kind: c.Kind, Series: c.Series, Version: d.Version, Module: c.Module, Fingerprint: c.Fingerprint, Commit: c.Commit, Revision: c.Revision, Digest: c.Digest, Seat: c.Seat, State: Numbered, At: v.now(), Merge: c.Merge} - if c.Tag != "" { - e.Repository = c.Repository - } raw, err := Encode(e) if err != nil { return Decision{}, Entry{}, err @@ -205,7 +182,7 @@ func fresh(entries []Entry, c Claim) []Entry { return out } -// All is every entry of every name, by name in order: what builder_untag and the tag asks read. +// All is every entry of every name, by name in order: what the versions' readers read. func (v Versions) All(ctx context.Context) ([]Entry, error) { keys, err := v.Store.KeysOf(ctx, AllFilter) if err != nil { diff --git a/modules/mesh-delivery-builder/internal/versions/tagbound.go b/modules/mesh-delivery-builder/internal/versions/tagbound.go deleted file mode 100644 index 51cfdc83..00000000 --- a/modules/mesh-delivery-builder/internal/versions/tagbound.go +++ /dev/null @@ -1,75 +0,0 @@ -package versions - -import ( - "fmt" - "maps" - "slices" - "strings" - "time" -) - -// The bound of an unanswered tag ask (design 51 *Tagged versions*: "an ask the builder has not answered within five -// minutes is narrated in the build log with what it waits on"): a key of the state tag-ask-bounds, written with its own -// time to live, whose expiry the builder hears on its watch of the state. No timer: the bus ends the bound. -const ( - TagAskBoundsState = "tag-ask-bounds" - TagAskBoundsBucket = "mesh-delivery-builder_tag-ask-bounds" - TagAskBound = 5 * time.Minute -) - -// Unanswered says whether a tag ask still waits for its versions: kept, not an untag, none numbered yet. -func (t TagAsk) Unanswered() bool { - return !t.Untag && t.State == TagAsked && len(t.Versions) == 0 -} - -// WithNumbered is the tag ask once a check's build of its head numbered module's packages under its tag: numbered, each -// version by the name it was numbered under (names, by artifact; NameOf where it names none), the answer naming them. An -// untag, or an ask of another head, is left as it is. -func (t TagAsk) WithNumbered(module, head string, got, names map[string]string, at time.Time) (TagAsk, bool) { - if t.Untag || t.Head != head || len(got) == 0 || t.State == TagRemoving || t.State == TagRemoved { - return t, false - } - if t.Versions == nil { - t.Versions = map[string]string{} - } - changed := false - for artifact, v := range got { - name := Numbering{Names: names}.NameOf(module, artifact) - if t.Versions[name] != v { - t.Versions[name], changed = v, true - } - } - if !changed && t.State == TagNumbered { - return t, false - } - var said []string - for _, name := range slices.Sorted(maps.Keys(t.Versions)) { - said = append(said, name+" "+t.Versions[name]) - } - t.State, t.Answered = TagNumbered, at.UTC() - t.Answer = "numbered: " + strings.Join(said, ", ") - return t, true -} - -// NarrateUnanswered is the line said for a tag ask its bound found unanswered: since when, and what it waits on, the -// names waiting on their floor named with their silent sources. -func NarrateUnanswered(t TagAsk, waits []FloorWait, now time.Time) string { - on := "the check's build of " + short(t.Head) + " is not claimed yet (the checker builds a group member's head once it asks)" - var floors []string - for _, w := range waits { - floors = append(floors, w.Name+" waits on its floor ("+strings.Join(w.Silent, ", ")+" silent)") - } - if len(floors) > 0 { - slices.Sort(floors) - on += "; " + strings.Join(floors, "; ") - } - return fmt.Sprintf("the tag ask of %s %s at %s (%s) is not answered after %s: it waits on %s", - t.Repository, t.Branch, short(t.Head), t.Tag, now.Sub(t.At).Round(time.Second), on) -} - -func short(head string) string { - if len(head) > 12 { - return head[:12] - } - return head -} diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index bb8d94f0..798bb0f8 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -6,8 +6,6 @@ import ( "errors" "fmt" "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" ) // The builder's tools of its versions (design 51 *Events of the builder*, *Tools of the builder*). @@ -15,21 +13,12 @@ const ( VersionsTool = "builder_versions" PublishAgainTool = "builder_publish_again" FloorAgainTool = "builder_floor_again" - TagTool = "builder_tag" - TagAsksTool = "builder_tag_asks" - UntagTool = "builder_untag" OldFormTool = "builder_old_form" - // OldFormClosedEvent is, with builds.VersionNumberedEvent, the builder's own family `version`, kept seven days, + // OldFormClosedEvent is the builder's own family `version`, kept seven days, // apart from its builds' ends, so a waiter on a build's end does not wake on them. OldFormClosedEvent = "version.old-form-closed" ) -// NumberedOf is the version.numbered body of an entry (builds.VersionNumbered, the contract a build module reads). -func NumberedOf(e Entry, head string) builds.VersionNumbered { - return builds.VersionNumbered{Name: e.Name, Kind: e.Kind, Version: e.Version, Tag: e.Tag, Module: e.Module, Fingerprint: e.Fingerprint, - Commit: e.Commit, Digest: e.Digest, Seat: e.Seat, Head: head} -} - // Reader reads the state as the tools bundle may: every key, and one. type Reader struct { Keys func() ([]string, error) diff --git a/modules/mesh-delivery-builder/internal/versions/versions.go b/modules/mesh-delivery-builder/internal/versions/versions.go index 4e129e31..8d6cf194 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions.go +++ b/modules/mesh-delivery-builder/internal/versions/versions.go @@ -302,47 +302,6 @@ func Decide(entries []Entry, kind, series, fingerprint, revision string, floor F return Decision{Version: v, Publish: true, Why: "numbered " + v} } -// CleanTag makes a branch or tag name a semver prerelease identifier: lower case, every run of characters other than -// letters and digits one '-', none at either end; "tag" when nothing is left. -func CleanTag(tag string) string { - var b strings.Builder - dash := false - for _, r := range strings.ToLower(tag) { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { - b.WriteRune(r) - dash = false - continue - } - if !dash && b.Len() > 0 { - b.WriteByte('-') - dash = true - } - } - out := strings.TrimSuffix(b.String(), "-") - if out == "" { - return "tag" - } - return out -} - -// Tagged is the next tagged version of name for tag: below the next trunk version, never the latest -// (`0.1.15-feat-x.2`, design 51 *Tagged versions*); a package new in a group is numbered from its floor -// (`0.1.0-feat-x.1`). -func Tagged(entries []Entry, series, tag string, floor Floor) string { - base := Next(entries, series, floor) - tag = CleanTag(tag) - n := 0 - prefix := base + "-" + tag + "." - for _, e := range entries { - if rest, ok := strings.CutPrefix(e.Version, prefix); ok { - if k, err := strconv.Atoi(rest); err == nil && k > n { - n = k - } - } - } - return fmt.Sprintf("%s%d", prefix, n+1) -} - // Fill is whether a bucket holding bytes of cap is past its finding's mark. func Fill(bytes, capacity uint64) (share float64, full bool) { if capacity == 0 { diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index 9bb80e17..59b9b7a3 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms" ) @@ -241,18 +240,6 @@ func TestTheBuildersVersions(t *testing.T) { t.Error("builder_versions with no name answered") } }) - t.Run("tagged", func(t *testing.T) { - es := []Entry{{Version: "0.1.14", Series: "0.1"}, {Version: "0.1.15-feat-x.1", Series: "0.1", Tag: "feat-x"}} - if got := Tagged(es, "0.1", "feat/X", known(nil, nil, nil)); got != "0.1.15-feat-x.2" { - t.Errorf("Tagged = %s, want 0.1.15-feat-x.2", got) - } - if got := Tagged(nil, "0.1", "feat-x", known(nil, nil, nil)); got != "0.1.0-feat-x.1" { - t.Errorf("a package new in a group = %s, want 0.1.0-feat-x.1", got) - } - if got := CleanTag("--Fix/ä_1965--"); got != "fix-1965" { - t.Errorf("CleanTag = %s", got) - } - }) t.Run("the floor wakes on its silent source, never a timer", func(t *testing.T) { ctx := context.Background() src := FloorSources{ @@ -400,17 +387,16 @@ func TestTheBuildersVersions(t *testing.T) { }, Requirements: func(context.Context, string) ([]string, error) { return nil, nil }, } - var said []builds.VersionNumbered fw := &FloorWaits{} - n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Waits: fw, Said: func(b builds.VersionNumbered) { said = append(said, b) }} + n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Waits: fw} c := BuildClaim{Module: "m", Path: "go", Repository: "novox/m", Fingerprint: "fp1", Commit: "c1", Revision: "7", Merge: 7, Artifacts: []ClaimArtifact{{Artifact: "program", Kind: KindBundle, Language: "go"}, {Artifact: "base", Kind: "upstream"}}} got, err := n.Number(ctx, c) if err != nil || len(got.Versions) != 0 || !reflect.DeepEqual(got.Unnumbered, []string{"program"}) || got.Wait != "" { t.Fatalf("silent floor: %+v, %v; want program unnumbered, the upstream copy left out", got, err) } - if !reflect.DeepEqual(asked, []string{"novox/m go/v"}) || len(fw.Waiting()) != 1 || len(said) != 0 { - t.Fatalf("asked %v, waits %+v, said %v", asked, fw.Waiting(), said) + if !reflect.DeepEqual(asked, []string{"novox/m go/v"}) || len(fw.Waiting()) != 1 { + t.Fatalf("asked %v, waits %+v", asked, fw.Waiting()) } pkg := c pkg.Artifacts = []ClaimArtifact{{Artifact: "lib", Kind: KindPackage, Language: "go"}} @@ -419,27 +405,9 @@ func TestTheBuildersVersions(t *testing.T) { } tagsUp = true got, err = n.Number(ctx, c) - if err != nil || got.Versions["program"] != "0.1.14" || len(said) != 1 || said[0].Version != "0.1.14" || len(fw.Waiting()) != 1 { - t.Fatalf("floor answered: %+v, %v, said %v, waits %+v; want 0.1.14 said once, lib still waiting", got, err, said, fw.Waiting()) + if err != nil || got.Versions["program"] != "0.1.14" || len(fw.Waiting()) != 1 { + t.Fatalf("floor answered: %+v, %v, waits %+v; want 0.1.14, lib still waiting", got, err, fw.Waiting()) } - // DLV-B6: a check's build of a head a tag ask names numbers only its packages, tagged below the next version, kept - // with its repository for the branch's end, said with the head; claimed again it keeps its number. - tagged := c - tagged.Tag, tagged.Head, tagged.Revision, tagged.Merge = "feat-x", "h1", "", 0 - tagged.Artifacts = append(tagged.Artifacts, ClaimArtifact{Artifact: "tlib", Kind: KindPackage, Language: "go"}) - saidBefore, askedBefore := said, asked - said = nil - got, err = n.Number(ctx, tagged) - if err != nil || len(got.Versions) != 1 || !strings.Contains(got.Versions["tlib"], "-feat-x.1") || len(said) != 1 || said[0].Head != "h1" || said[0].Tag != "feat-x" { - t.Fatalf("tagged claim: %+v, %v, said %+v; want tlib alone tagged, said with its head", got, err, said) - } - if es, _ := n.Versions.Of(ctx, "m/tlib"); len(es) != 1 || es[0].Repository != "novox/m" { - t.Errorf("the tagged version kept = %+v, want its repository", es) - } - if again, _ := n.Number(ctx, tagged); again.Versions["tlib"] != got.Versions["tlib"] || len(said) != 1 { - t.Errorf("claimed again: %+v, said %d; want the same number, said once", again, len(said)) - } - said, asked = saidBefore, askedBefore got, _ = n.Number(ctx, c) var publishing []string // review round 1: a version built and still numbered arms its publish bound n.Publishing = func(_ context.Context, e Entry) { publishing = append(publishing, e.Name+"@"+e.Version) } @@ -450,8 +418,8 @@ func TestTheBuildersVersions(t *testing.T) { t.Fatalf("the publish bounds armed: %v, want m/program@0.1.14", publishing) } again, _ := n.Number(ctx, c) - if again.Versions["program"] != "0.1.14" || len(said) != 1 || len(asked) != 3 { - t.Fatalf("again: %+v, said %d, asked %v; want the number kept, nothing said, the floor not asked", again, len(said), asked) + if again.Versions["program"] != "0.1.14" || len(asked) != 3 { + t.Fatalf("again: %+v, asked %v; want the number kept, the floor not asked", again, asked) } if _, ok, why, _ := n.Versions.Recorded(ctx, "m/program", "0.1.14", "sha256:bb"); ok { t.Fatalf("another digest recorded: %s", why) @@ -510,8 +478,7 @@ func TestTheBuildersVersions(t *testing.T) { return nil, nil }, } - var said []builds.VersionNumbered - n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Names: NamesIn(state), Said: func(b builds.VersionNumbered) { said = append(said, b) }} + n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Names: NamesIn(state)} c := BuildClaim{Module: "mesh-sdk", Path: "go", Repository: "novox/mesh-sdk", Fingerprint: "fp1", Commit: "c1", Revision: "3", Merge: 3, Artifacts: []ClaimArtifact{{Artifact: "go", Kind: KindPackage, Language: "go"}, {Artifact: "ts", Kind: KindPackage, Language: "typescript"}, {Artifact: "new", Kind: KindPackage, Language: "go"}, {Artifact: "program", Kind: KindBundle, Language: "go"}}} @@ -531,41 +498,26 @@ func TestTheBuildersVersions(t *testing.T) { if es, _ := n.Versions.Of(ctx, "mesh-sdk/go"); len(es) != 0 { t.Errorf("numbered under the builder's name too: %+v", es) } - names := map[string]bool{} - for _, b := range said { - names[b.Name] = true - } - if !names["git.novox.be/novox/mesh-sdk/go"] || !names["@novox/mesh-sdk"] { - t.Errorf("version.numbered said %v, want the ecosystem names", names) - } if err := n.Built(ctx, c, got, map[string]string{"go": "sha256:aa"}); err != nil { t.Fatal(err) } if es, _ := n.Versions.Of(ctx, "git.novox.be/novox/mesh-sdk/go"); len(es) != 1 || es[0].Digest != "sha256:aa" { t.Errorf("the built digest is not recorded under the name: %+v", es) } - tagged, _ := TagAsk{Head: "h", State: TagAsked}.WithNumbered("mesh-sdk", "h", map[string]string{"go": "0.1.4-x.1"}, got.Names, time.Unix(1, 0)) - if tagged.Versions["git.novox.be/novox/mesh-sdk/go"] != "0.1.4-x.1" { - t.Errorf("a tag ask names %v, want the Go module path", tagged.Versions) - } // The indexer writes the name of `new` later: its series key names the module, and the look at the done build - // numbers it with the done key's digest, said in version.numbered for the late publish; the rest are kept. + // numbers it with the done key's digest; the rest are kept. state["new.0.1"] = []byte(`{"v":1,"name":"new","package":"mesh-sdk/new","series":"0.1"}`) for key, want := range map[string]string{"new.0.1": "mesh-sdk", "git=2enovox=2ebe/novox/mesh-sdk/go": "", "broken.0.1": ""} { if module, name, ok := NamedModule(key, state[key]); module != want || ok != (want != "") || (ok && name != "new") { t.Errorf("NamedModule(%s) = %q, %q, %v, want %q", key, module, name, ok, want) } } - said = nil done := c done.Done = map[string]string{"go": "sha256:aa", "new": "sha256:bb"} late, err := n.Number(ctx, done) if err != nil || late.Versions["new"] != "0.1.4" || late.Versions["go"] != "0.1.4" || len(late.Unnumbered) != 0 { t.Fatalf("a look at the done build = %+v, %v; want new numbered and go kept", late, err) } - if len(said) != 1 || said[0].Name != "new" || said[0].Digest != "sha256:bb" { - t.Errorf("version.numbered said %+v, want new once with the done build's digest", said) - } if es, _ := n.Versions.Of(ctx, "new"); len(es) != 1 || es[0].Digest != "sha256:bb" { t.Errorf("versions of new = %+v, want its done digest recorded", es) } @@ -657,23 +609,25 @@ func TestTheBuildersVersions(t *testing.T) { t.Error("a version refused for other bytes is published again") } _, _ = v.Mark(ctx, "m/p", "0.1.4", func(e *Entry) { e.State = Published }) - var said []builds.VersionNumbered var rebuilt []string done := true - acts := Acts{Numberer: &Numberer{Versions: v, Said: func(b builds.VersionNumbered) { said = append(said, b) }, Waits: &FloorWaits{}}, + acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}}, Done: func(context.Context, string, string) (bool, error) { return done, nil }, BuildAgain: func(_ context.Context, m, fp, _, _ string) (string, bool) { rebuilt = append(rebuilt, m+" "+fp) return "built again", false }, Reconsider: func(context.Context) {}} - if ag, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(5, 0)); err != nil || ag.Do != AgainRenumber || ag.Entry.Version != "0.1.5" || - len(said) != 1 || said[0].Version != "0.1.5" || said[0].Again == "" || len(rebuilt) != 0 { - t.Errorf("publish again of a published, held build: %+v %v said %+v rebuilt %v", ag, err, said, rebuilt) + // Nothing says a version numbered (design 47 G §349): a held build is refused before any number is taken. + if _, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(5, 0)); err == nil || len(rebuilt) != 0 { + t.Errorf("publish again of a published, held build answered %v, rebuilt %v; want a refusal", err, rebuilt) + } + if es, _ := v.Of(ctx, "m/p"); len(es) != 2 { + t.Errorf("the refused publish again took a number: %+v", es) } done = false - if ag, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(6, 0)); err != nil || ag.Do != AgainBuild || ag.Entry.Version != "0.1.5" || len(rebuilt) != 1 || len(said) != 1 { - t.Errorf("publish again of a gone build: %+v %v said %d rebuilt %v", ag, err, len(said), rebuilt) + if ag, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(6, 0)); err != nil || ag.Do != AgainRenumber || ag.Entry.Version != "0.1.5" || len(rebuilt) != 1 { + t.Errorf("publish again of a gone build: %+v %v rebuilt %v", ag, err, rebuilt) } if _, err := acts.FloorAgain(ctx, "m/p"); err == nil { t.Error("a name with versions asks its floor again") @@ -682,147 +636,4 @@ func TestTheBuildersVersions(t *testing.T) { t.Errorf("floor again of a new name: %q %v", ans, err) } }) - t.Run("tag asks", func(t *testing.T) { - at := time.Unix(100, 0) - k, raw, err := NewTagAsk("novox/mesh-sdk", "feat/X y", "abc", "a check", "me@shanks", at) - if err != nil || k != TagAskKey("novox/mesh-sdk", "feat/X y") || strings.Count(k, ".") != 1 { - t.Fatalf("NewTagAsk = %q, %v", k, err) - } - a, err := DecodeTagAsk(raw) - if err != nil || a.Tag != "feat-x-y" || a.State != TagAsked || a.Untag { - t.Fatalf("DecodeTagAsk = %+v, %v", a, err) - } - for _, bad := range [][4]string{{"mesh-sdk", "b", "h", "w"}, {"o/r", "", "h", "w"}, {"o/r", "b", "", "w"}, {"o/r", "b", "h", ""}} { - if _, _, err := NewTagAsk(bad[0], bad[1], bad[2], bad[3], "", at); err == nil { - t.Errorf("NewTagAsk%v is not refused", bad) - } - } - _, uraw, err := NewUntag("o/r", "feat/x", "branch gone by hand", "", at.Add(time.Second)) - if u, _ := DecodeTagAsk(uraw); err != nil || !u.Untag { - t.Fatalf("NewUntag = %+v, %v", u, err) - } - vals := map[string][]byte{"a": uraw, "b": raw} - r := Reader{Keys: func() ([]string, error) { return []string{"a", "b"}, nil }, - Get: func(k string) (json.RawMessage, bool, error) { return vals[k], true, nil }} - if got, err := AnswerTagAsks(r, ""); err != nil || len(got.Asks) != 2 || got.Asks[0].Repository != "novox/mesh-sdk" { - t.Errorf("AnswerTagAsks = %+v, %v", got, err) - } - if got, _ := AnswerTagAsks(r, "o/r"); len(got.Asks) != 1 || !got.Asks[0].Untag { - t.Errorf("AnswerTagAsks(o/r) = %+v", got) - } - es := []Entry{{Name: "n", Version: "0.1.2-feat-x.1", Tag: "feat-x", Series: "0.1", Repository: "o/r"}, {Name: "n", Version: "0.1.2-feat-x.2", Tag: "feat-x", Series: "0.1", State: Removed, Repository: "o/r"}, - {Name: "n2", Version: "0.1.2-feat-x.1", Tag: "feat-x", Series: "0.1", Repository: "o/other"}, - {Name: "n", Version: "0.1.1", Series: "0.1"}, {Name: "n", Version: "0.1.2-feat-y.1", Tag: "feat-y", Series: "0.1"}} - for _, c := range []struct { - asks []TagAsk - repo, head string - want string - }{ - {[]TagAsk{a}, "novox/mesh-sdk", "abc", "feat-x-y"}, - {[]TagAsk{a}, "novox/mesh-sdk", "other", ""}, - {[]TagAsk{{Repository: "novox/mesh-sdk", Head: "abc", Tag: "t", State: TagRemoved}}, "novox/mesh-sdk", "abc", ""}, - } { - if got, _ := TagFor(c.asks, c.repo, c.head); got != c.want { - t.Errorf("TagFor(%s, %s) = %q, want %q", c.repo, c.head, got, c.want) - } - } - if got := Untagged(es, "o/r", "feat/x"); len(got) != 1 || got[0].Version != "0.1.2-feat-x.1" { - t.Errorf("Untagged = %+v", got) - } - st := newMem() - for _, e := range append(es, Entry{Name: "n3", Version: "0.1.0-feat-x.1", Tag: "feat-x", Series: "0.1", Repository: "o/r", Seat: SeatNPM}) { - e.Kind, e.Module, e.Fingerprint, e.At = KindPackage, "m", "f", at - if e.State == "" { - e.State = Numbered - } - raw, err := Encode(e) - if err != nil { - t.Fatal(err) - } - _, _ = st.Create(context.Background(), Key(e.Name, e.Version), raw) - } - var removed []string - var kept map[string]bool // review round 2: the names whose seat keeps them whole (a shared digest) - acts := Acts{Numberer: &Numberer{Versions: Versions{Store: st}}, Remove: func(_ context.Context, seat string, e Entry) (RemoveOutcome, error) { - removed = append(removed, e.Name+"@"+e.Version) - if kept[e.Name] { - return RemoveOutcome{Kept: true, Reason: "shared-digest", Tags: []string{"0.1.2"}}, nil - } - if seat == SeatNPM { - return RemoveOutcome{}, errors.New("down") - } - return RemoveOutcome{}, nil - }} - u, _ := DecodeTagAsk(uraw) - got := acts.TagAsked(context.Background(), u, at) - if got.State != TagRemoving || len(removed) != 2 || len(got.Left) != 2 || !strings.Contains(got.Left["n3@0.1.0-feat-x.1"], "down") || got.Answered.IsZero() { - t.Errorf("untag: %+v removed %v", got, removed) - } - if k := acts.TagAsked(context.Background(), a, at); k.State != TagAsked || k.Answer == "" { - t.Errorf("a tag ask is not kept: %+v", k) - } - // DLV-B6: a branch's end asks remove of every tagged version not removed, and is done once each seat said removed. - removed = nil - end, err := acts.BranchEnded(context.Background(), "o/r", "feat/x") - if err != nil || end.Done || len(removed) != 2 || !reflect.DeepEqual(end.Seats(), []string{"", SeatNPM}) || !strings.Contains(end.Finding("o/r", "feat/x", at), "n3@0.1.0-feat-x.1 on npm-package-registry: down") { - t.Fatalf("BranchEnded = %+v %v, removed %v", end, err, removed) - } - for _, v := range [][2]string{{"n", "0.1.2-feat-x.1"}, {"n3", "0.1.0-feat-x.1"}} { - if _, err := acts.Numberer.Heard(context.Background(), SeatNPM, HeardRemoved, "rm", Outcome{Name: v[0], Version: v[1], Reason: "tag-gone"}); err != nil { - t.Fatal(err) - } - } - if end, _ := acts.BranchEnded(context.Background(), "o/r", "feat/x"); !end.Done { - t.Errorf("every seat said removed, and the branch's end is not done: %+v", end) - } - // Review round 2: a remove its seat answers kept (another tag names the digest) ends there, with no retry: the - // version is recorded kept on the store with the reason and the other tags, which builder_versions shows. - raw, _ = Encode(Entry{Name: "n4", Kind: KindPackage, Series: "0.1", Version: "0.1.2-feat-x.1", Tag: "feat-x", Repository: "o/r", Seat: SeatNPM, State: Numbered}) - _, _ = st.Create(context.Background(), Key("n4", "0.1.2-feat-x.1"), raw) - kept, removed = map[string]bool{"n4": true}, nil - if end, err := acts.BranchEnded(context.Background(), "o/r", "feat/x"); err != nil || !end.Done || len(removed) != 1 { - t.Fatalf("a kept remove: %+v %v, removed %v; want done at once", end, err, removed) - } - if es, _ := acts.Numberer.Versions.Of(context.Background(), "n4"); len(es) != 1 || es[0].State != Kept || es[0].Reason != "shared-digest" || !reflect.DeepEqual(es[0].KeptWith, []string{"0.1.2"}) { - t.Errorf("the kept version = %+v, want kept, shared-digest, with 0.1.2", es) - } - if removed = nil; func() bool { - end, _ := acts.BranchEnded(context.Background(), "o/r", "feat/x") - return !end.Done || len(removed) != 0 - }() { - t.Errorf("a kept version is asked again: removed %v", removed) - } - if _, err := acts.Numberer.Heard(context.Background(), SeatNPM, HeardKept, "k", Outcome{Name: "n", Version: "0.1.2-feat-x.1", Tags: []string{"0.1.2"}}); err != nil { - t.Fatal(err) - } - if es, _ := acts.Numberer.Versions.Of(context.Background(), "n"); slices.ContainsFunc(es, func(e Entry) bool { return e.Version == "0.1.2-feat-x.1" && e.State == Kept }) { - t.Error("a seat's kept turned a removed version back to kept") - } - // B6: a check's build of the ask's head marks it numbered with its versions; another head, or an untag, is left. - if !a.Unanswered() { - t.Errorf("a kept ask with no version is answered: %+v", a) - } - if _, changed := a.WithNumbered("mesh-sdk", "other", map[string]string{"go": "0.1.15-feat-x-y.1"}, nil, at); changed { - t.Error("an ask is marked numbered by a build of another head") - } - if _, changed := u.WithNumbered("o", "", map[string]string{"go": "1"}, nil, at); changed { - t.Error("an untag is marked numbered") - } - n, changed := a.WithNumbered("mesh-sdk", "abc", map[string]string{"go": "0.1.15-feat-x-y.1"}, nil, at.Add(time.Minute)) - if !changed || n.State != TagNumbered || n.Versions["mesh-sdk/go"] != "0.1.15-feat-x-y.1" || n.Unanswered() || !strings.Contains(n.Answer, "mesh-sdk/go 0.1.15-feat-x-y.1") { - t.Errorf("WithNumbered = %+v, %v", n, changed) - } - if _, again := n.WithNumbered("mesh-sdk", "abc", map[string]string{"go": "0.1.15-feat-x-y.1"}, nil, at); again { - t.Error("the same versions mark the ask again") - } - said := NarrateUnanswered(a, []FloorWait{{Name: "mesh-sdk/go", Silent: []string{"bridge_tags"}}}, at.Add(6*time.Minute)) - for _, w := range []string{"novox/mesh-sdk feat/X y", "after 6m0s", "not claimed yet", "mesh-sdk/go waits on its floor (bridge_tags silent)"} { - if !strings.Contains(said, w) { - t.Errorf("NarrateUnanswered = %q, want %q in it", said, w) - } - } - if k, v, err := TagWantedAsk(TagWanted{Repository: "o/r", Branch: "feat/x", Head: "abc"}, "mesh-delivery-checker", at); err != nil || k != TagAskKey("o/r", "feat/x") || !strings.Contains(string(v), `"by":"mesh-delivery-checker"`) { - t.Errorf("TagWantedAsk = %q %s %v", k, v, err) - } - }) } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 6fe5c6b9..5c6c1068 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -102,7 +102,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-ask-bounds", History: 1, KeyTTL: true}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -146,7 +146,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"container-image-registry.kept", "container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed", "go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed", "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", - "mesh-conditions.condition-cleared", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", + "mesh-conditions.condition-cleared", "mesh-delivery-assigner.views.assignments", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "node-engine.manifest-reads-changed", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", @@ -161,11 +161,11 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_tag", "builder_tag_asks", "builder_untag", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - // Design 51 *Events of the builder*: numbered and old-form-closed on the family version, apart from the builds' ends. - if want := []string{"build.asked-again", "version.numbered", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { + // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. + if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } // 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days in a cap that holds them; no bare stream. diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 8ed6a82d..f3d82244 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -26,9 +26,6 @@ "builder_reserved_drop", "builder_seat_queue", "builder_stuck", - "builder_tag", - "builder_tag_asks", - "builder_untag", "builder_versions", "reserved", "builder_first_views", @@ -36,7 +33,6 @@ ], "emits": [ "build.asked-again", - "version.numbered", "version.old-form-closed" ], "consumes": [ @@ -55,7 +51,6 @@ "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", - "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", "node-engine.manifest-reads-changed", "npm-package-registry.kept", @@ -110,8 +105,6 @@ { "name": "queues", "history": 1 }, { "name": "reserved", "history": 1 }, { "name": "reasks", "history": 1 }, - { "name": "tag-ask-bounds", "history": 1, "key-ttl": true }, - { "name": "tag-asks", "history": 1 }, { "name": "versions", "history": 1 }, { "name": "first-views", "history": 1 } ], From 18f978e85203a47b5589cd1ff8a46b7751b11b86 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:49:02 +0100 Subject: [PATCH 8/9] BL8: the builder hears the build seats' toolchain-reported, keeps them in toolchains-heard and says toolchain-agreed with digests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder agrees a language's toolchain from the three build seats' toolchain-reported, kept in its state toolchains-heard (read back at its start), forgets a node's on the node-engine seat's running with no fingerprint (stopped) and never on a placement, says toolchain-agreed with its digests list only when the agreed version changes; builder_toolchains and builder_toolchains_drop; the manifest's reads of the build modules' toolchains are gone. The gate over the assignments is now PlacedNodes, which only counts nodes. Builds: design 47 G §492, §604, §546, §472, §574 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-builder/buildrun/names.go | 3 - .../buildrun/standing.go | 208 ++--------- .../buildrun/standing_test.go | 239 ++---------- .../buildrun/toolchains.go | 91 +++-- .../buildrun/toolchains_test.go | 51 ++- .../cmd/mesh-delivery-builder-tools/main.go | 1 + .../mesh-delivery-builder-tools/toolchains.go | 82 +++++ .../toolchains_test.go | 73 ++++ .../gone_integration_test.go | 2 +- .../mesh-delivery-builder/integration_test.go | 39 +- .../cmd/mesh-delivery-builder/main.go | 43 ++- .../cmd/mesh-delivery-builder/names_test.go | 4 +- .../cmd/mesh-delivery-builder/reads.go | 4 +- .../cmd/mesh-delivery-builder/refused_test.go | 16 +- .../cmd/mesh-delivery-builder/toolchains.go | 343 ++++++++++++++---- .../mesh-delivery-builder/toolchains_test.go | 153 ++++++++ .../cmd/mesh-delivery-builder/versions.go | 19 + modules/mesh-delivery-builder/go.mod | 2 + .../internal/build/fakes_test.go | 11 +- .../internal/build/run_test.go | 20 +- .../mesh-delivery-builder/manifest_test.go | 14 +- modules/mesh-delivery-builder/module.json | 10 +- 22 files changed, 818 insertions(+), 610 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains_test.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go diff --git a/modules/mesh-delivery-builder/buildrun/names.go b/modules/mesh-delivery-builder/buildrun/names.go index 50ab553c..f42177cf 100644 --- a/modules/mesh-delivery-builder/buildrun/names.go +++ b/modules/mesh-delivery-builder/buildrun/names.go @@ -11,6 +11,3 @@ const ( // StepsBucket is the bucket of a build module's state steps. func StepsBucket(module string) string { return module + "_steps" } - -// ToolchainsBucket is the bucket of a build module's state toolchains. -func ToolchainsBucket(module string) string { return module + "_toolchains" } diff --git a/modules/mesh-delivery-builder/buildrun/standing.go b/modules/mesh-delivery-builder/buildrun/standing.go index f97ae7c0..52a490b1 100644 --- a/modules/mesh-delivery-builder/buildrun/standing.go +++ b/modules/mesh-delivery-builder/buildrun/standing.go @@ -3,9 +3,6 @@ package buildrun import ( "context" "errors" - "fmt" - "maps" - "slices" "sort" "sync" "time" @@ -15,120 +12,60 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/process" ) -// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key -// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the -// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left -// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). -// -// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a -// ToolchainSet): an entry is passed on only while the assignments (the seat mesh-module-assigner's state `assignments`, held by -// the assigner, AssignmentsBucket, read by meshmoduleassignerclient.Placed; ADR 0567, ADR 0549 §3) places its build module on -// its node, and only once it was written since that assignments; it is passed on as deleted otherwise, said once, and -// passed on when its node writes it again. A report written before the assignments is one an earlier run of the module -// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23 -// back into the agreement before the module ran there). Until the assignments is settled, and while no node's assignments -// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed -// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465, -// and it stays. -// -// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's -// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6). -type ToolchainGate struct { - mu sync.Mutex - modules map[string]bool // the build modules whose toolchains pass the gate - forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds - logf func(format string, a ...any) - placed placements - known bool // the assignments is settled - raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it - left map[string]string // / -> why it was said left out, so each reason is said once - // OnPlaced is told, outside the gate's lock, once the assignments's history is read and after each revision of it +// PlacedNodes holds which modules the catalogue places on which node: the seat mesh-module-assigner's state `assignments`, +// held by the assigner (AssignmentsBucket, read by meshmoduleassignerclient.Placed; ADR 0567), or the assignment half +// (meshwantsclient.Placed, fed by Revision). The builder counts the nodes it names (Nodes; ADR 0624 §1). It holds no +// toolchain: the builder keeps the toolchains it heard in its state toolchains-heard and forgets one when the node-engine +// seat says the build module stopped on the node, never on a placement (design 47 G §546). +type PlacedNodes struct { + mu sync.Mutex + logf func(format string, a ...any) + placed placements + known bool // the assignments is settled + // OnPlaced is told, outside the lock, once the assignments's history is read and after each revision of it // FollowAssignments hears from then on (the counted nodes, Nodes, may have changed). Nil tells nobody. OnPlaced func() } -// placements is where the gate reads which modules stand on which node, and since when: the assigner's state -// (meshmoduleassignerclient.Placed, fed by Assignments) or the assignment half (meshwantsclient.Placed, fed by Revision). +// placements is where the part reads which modules stand on which node. type placements interface { Seeded() bool - Since(node, module string) (time.Time, bool) ByNode() map[string][]string } -// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what -// it leaves out. -func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate { - g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: meshmoduleassignerclient.NewPlaced(), - raw: map[string]map[string][]byte{}, left: map[string]string{}} - for _, m := range modules { - g.modules[m] = true - } - return g +// NewPlacedNodes is the placements of the builder, saying with logf what it cannot read. +func NewPlacedNodes(logf func(string, ...any)) *PlacedNodes { + return &PlacedNodes{logf: logf, placed: meshmoduleassignerclient.NewPlaced()} } -// UseModules replaces the build modules whose reports the gate passes on again when the assignments moves (a part that -// finds them from the seat list and the manifests, as the indexer does, gives them at each change), and passes every -// report it holds on again. It answers whether what it passed on changed. -func (g *ToolchainGate) UseModules(modules []string) bool { - g.mu.Lock() - defer g.mu.Unlock() - g.modules = map[string]bool{} - for _, m := range modules { - g.modules[m] = true - } - return g.all() -} - -// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. -func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { - g.mu.Lock() - defer g.mu.Unlock() - if g.raw[module] == nil { - g.raw[module] = map[string][]byte{} - } - if deleted { - delete(g.raw[module], key) - } else { - g.raw[module][key] = value - } - return g.pass(module, key) -} - -// Assignments takes one revision of the seat mesh-module-assigner's assignments in (at is when the bus wrote it) and answers whether what it -// passed on changed. A value that does not read is said, and places nothing on its node. -func (g *ToolchainGate) Assignments(key string, value []byte, deleted bool, at time.Time) bool { +// Assignments takes one revision of the seat mesh-module-assigner's assignments in (at is when the bus wrote it) and +// answers whether the nodes the modules stand on changed. A value that does not read is said, and places nothing. +func (g *PlacedNodes) Assignments(key string, value []byte, deleted bool, at time.Time) bool { g.mu.Lock() defer g.mu.Unlock() placed, ok := g.placed.(*meshmoduleassignerclient.Placed) if !ok { - g.logf("%s: %s is not read, the gate takes the assignment half (design 47)", AssignmentsBucket, key) + g.logf("%s: %s is not read, the placements take the assignment half (design 47)", AssignmentsBucket, key) return false } was := placed.Seeded() - node, changed, err := placed.Put(key, value, deleted, at) + _, changed, err := placed.Put(key, value, deleted, at) if err != nil { g.logf("%s: %v (issue 1940)", AssignmentsBucket, err) } if !g.known { return false } - if was != g.placed.Seeded() { - // The first assignments came, or the last went: every report's standing changed. - return g.all() - } - return g.again(node, changed) + return was != g.placed.Seeded() || len(changed) > 0 } // Revision takes one assignment graph revision the part's view holds in (design 47 *Each part acts on its own entries*: -// the placements come from the seat mesh-wants' assignment half, never the assigner's state; assignmentview's Held hands -// each) and answers whether what it passed on changed. The first revision taken settles the gate, as the assignments' -// history read does; a later one passes on again the reports of each node whose build modules' standing changed. A gate -// fed by Revision reads no Assignments. -func (g *ToolchainGate) Revision(r meshwants.AssignmentRevision) bool { +// the placements come from the seat mesh-wants' assignment half, never the assigner's state) and answers whether the +// nodes the modules stand on changed. The first revision taken settles it. Placements fed by Revision read no Assignments. +func (g *PlacedNodes) Revision(r meshwants.AssignmentRevision) bool { g.mu.Lock() defer g.mu.Unlock() placed, ok := g.placed.(*meshwantsclient.Placed) @@ -141,20 +78,14 @@ func (g *ToolchainGate) Revision(r meshwants.AssignmentRevision) bool { if changed == nil { return false } - if !g.known || was != placed.Seeded() { - g.known = true - return g.all() - } - moved := false - for _, node := range slices.Sorted(maps.Keys(changed)) { - moved = g.again(node, changed[node]) || moved - } - return moved + first := !g.known + g.known = true + return first || was != placed.Seeded() || len(changed) > 0 } // Nodes are the nodes the assignments places any module on, sorted: the nodes the mesh counts (a node leaves the count // when the assignments no longer names it; ADR 0624 §1). None until the assignments is read. -func (g *ToolchainGate) Nodes() []string { +func (g *PlacedNodes) Nodes() []string { var out []string for node, modules := range g.placed.ByNode() { if len(modules) > 0 { @@ -165,95 +96,27 @@ func (g *ToolchainGate) Nodes() []string { return out } -// Settled says the assignments's history is read: from now on only the reports of placed nodes pass, once a node's -// assignments is held. It answers whether what it passed on changed. -func (g *ToolchainGate) Settled() bool { +// Settled says the assignments's history is read. +func (g *PlacedNodes) Settled() { g.mu.Lock() defer g.mu.Unlock() g.known = true - return g.all() -} - -// all passes on every report again. The caller holds mu. -func (g *ToolchainGate) all() bool { - changed := false - for _, module := range slices.Sorted(maps.Keys(g.raw)) { - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - changed = g.pass(module, k) || changed - } - } - return changed -} - -// again passes on every report of node by a build module among modules. The caller holds mu. -func (g *ToolchainGate) again(node string, modules []string) bool { - changed := false - for _, module := range modules { - if !g.modules[module] { - continue - } - for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { - if n, _, err := buildstep.ParseToolchainKey(k); err == nil && n == node { - changed = g.pass(module, k) || changed - } - } - } - return changed -} - -// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the assignments is not -// known), deleted otherwise. The caller holds mu. -func (g *ToolchainGate) pass(module, key string) bool { - value, held := g.raw[module][key] - if !held { - return g.forward(module, key, nil, true) - } - node, _, err := buildstep.ParseToolchainKey(key) - why := "" - if err == nil && g.known && g.placed.Seeded() { - why = g.leftOut(module, node, value) - } - if why == "" { - delete(g.left, module+"/"+key) - return g.forward(module, key, value, false) - } - if g.left[module+"/"+key] != why { - g.left[module+"/"+key] = why - g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", ToolchainsBucket(module), key, why) - } - return g.forward(module, key, nil, true) -} - -// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its -// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse. -// The caller holds mu. -func (g *ToolchainGate) leftOut(module, node string, value []byte) string { - since, placed := g.placed.Since(node, module) - if !placed { - return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node) - } - tc, err := buildstep.DecodeToolchain(value) - if err == nil && tc.At.Before(since) { - return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again", - tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) - } - return "" } // FollowAssignments reads the seat mesh-module-assigner's assignments with its history into gate, says it settled, and follows it; changed // is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part // may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that // stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel. -func FollowAssignments(ctx context.Context, js process.BucketOpener, gate *ToolchainGate, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) { +func FollowAssignments(ctx context.Context, js process.BucketOpener, gate *PlacedNodes, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) { errs := make(chan error, 1) kv, err := js.KeyValue(ctx, AssignmentsBucket) if err != nil { - logf("the state %s cannot be read, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", AssignmentsBucket, err) + logf("the state %s cannot be read, so no node is counted by the assignments: %v", AssignmentsBucket, err) return errs, nil } w, err := kv.WatchAll(ctx, jetstream.IncludeHistory()) if err != nil { - logf("the state %s cannot be watched, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", AssignmentsBucket, err) + logf("the state %s cannot be watched, so no node is counted by the assignments: %v", AssignmentsBucket, err) return errs, nil } put := func(e jetstream.KeyValueEntry) bool { @@ -275,9 +138,8 @@ func FollowAssignments(ctx context.Context, js process.BucketOpener, gate *Toolc put(e) } } - if gate.Settled() { - changed(ctx) - } + gate.Settled() + changed(ctx) if gate.OnPlaced != nil { gate.OnPlaced() } diff --git a/modules/mesh-delivery-builder/buildrun/standing_test.go b/modules/mesh-delivery-builder/buildrun/standing_test.go index 5fca15d6..2b8da16f 100644 --- a/modules/mesh-delivery-builder/buildrun/standing_test.go +++ b/modules/mesh-delivery-builder/buildrun/standing_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "fmt" "log" "strings" "testing" @@ -15,19 +14,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) -func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) { - t.Helper() - key, _ := buildstep.ToolchainKey(node, "typescript") - value, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) - if err != nil { - t.Fatal(err) - } - return key, value -} - // placing is a value of the state `assignments` placing modules on node. func placing(t *testing.T, node string, modules ...string) (string, []byte) { t.Helper() @@ -43,219 +31,52 @@ func placing(t *testing.T, node string, modules ...string) (string, []byte) { return key, values[key] } -func agreedOf(set *ToolchainSet) (string, string) { - tc, none := buildstep.AgreedToolchain(set.Entries("typescript")) - return tc.Version, none -} - -// Issue 1940, the gate over the catalogue's assignments (issue 774, ADR 0549 §3): a build node's report counts only while -// the assignments places its build module on its node, and only once written since; before the assignments is settled, and -// while no node's assignments is held, every report counts. The live cases: mesh-build-node taken off ace left -// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came -// back before the module ran there. -func TestToolchainGate(t *testing.T) { - old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run - placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again +// The placements the builder counts nodes by (ADR 0624 §1): a settled assignments names the nodes its modules stand on, a +// revision of the assignment half the same; they hold no toolchain (design 47 G §546: a toolchain is forgotten on the +// node-engine seat's stopped, never on a placement). +func TestPlacedNodes(t *testing.T) { + old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) three := []string{"g14", "novox", "shanks"} - start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) { - set := &ToolchainSet{} - said := &[]string{} - gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) + + t.Run("the assignments names the counted nodes once it is settled", func(t *testing.T) { + gate := NewPlacedNodes(func(string, ...any) {}) for _, node := range three { - if seeded { - key, value := placing(t, node, NodeBuildModule) - gate.Assignments(key, value, false, old.Add(-time.Hour)) + k, v := placing(t, node, NodeBuildModule) + if gate.Assignments(k, v, false, old) { + t.Fatal("an assignments before it is settled changed the counted nodes") } - key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute)) - gate.Toolchain(NodeBuildModule, key, value, false) } - return gate, set, said - } - - t.Run("not placed on ace: its report is no builder's", func(t *testing.T) { - gate, set, said := start(t, true) - key, value := gateReport(t, "ace", "v26.12.0", old) - gate.Toolchain(NodeBuildModule, key, value, false) - if a := buildstep.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out - t.Fatalf("before the assignments is settled every report counts; got %+v", a) - } - k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter - gate.Assignments(k, v, false, old) - if !gate.Settled() { - t.Fatal("settling left ace's report out, so what was passed on changed") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none) - } - if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") { - t.Fatalf("ace's report is said left out once; said %q", *said) - } - if gate.Toolchain(NodeBuildModule, key, value, false) { - t.Fatal("a report of a node not placed changed what was passed on") - } - // Placed on ace with a report written since: it counts, and the builders really disagree. - k, v = placing(t, "ace", GoBuildModule, NodeBuildModule) - gate.Assignments(k, v, false, old.Add(-time.Minute)) - if a := buildstep.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" { - t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a) - } - // Taken off ace again (its key deleted): its report goes again. - if !gate.Assignments(k, nil, true, placed) { - t.Fatal("taking ace's assignments away passed nothing on") - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("got %q, %q", v, none) - } - // A report deleted from the bucket goes whether its node is placed or not. - key, _ = gateReport(t, "g14", "v26.12.1", placed) - if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 { - t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript"))) - } - }) - - t.Run("build modules given later pass their reports on when the assignments moves", func(t *testing.T) { - set := &ToolchainSet{} - gate := NewToolchainGate(nil, set.Put, func(string, ...any) {}) gate.Settled() - k, v := placing(t, "novox", GoBuildModule) - gate.Assignments(k, v, false, old) - key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) - gate.Toolchain(NodeBuildModule, key, value, false) // ace is not placed: left out - k, v = placing(t, "ace", NodeBuildModule) - if gate.Assignments(k, v, false, placed) || len(set.Entries("typescript")) != 0 { - t.Fatalf("a module the gate was not given passed on at the assignments: %v", set.Entries("typescript")) + if got := strings.Join(gate.Nodes(), ","); got != "g14,novox,shanks" { + t.Fatalf("nodes = %q", got) } - if !gate.UseModules([]string{NodeBuildModule}) || len(set.Entries("typescript")) != 1 { - t.Fatalf("given mesh-build-node, ace's report placed since is passed on; entries %v", set.Entries("typescript")) + k, v := placing(t, "ace", GoBuildModule) + if !gate.Assignments(k, v, false, old) || len(gate.Nodes()) != 4 { + t.Fatalf("ace placed: nodes %v", gate.Nodes()) + } + if !gate.Assignments(k, nil, true, old.Add(time.Hour)) || len(gate.Nodes()) != 3 { + t.Fatalf("ace's assignments deleted: nodes %v", gate.Nodes()) } }) - for _, c := range []struct { - name string - at time.Time // when ace wrote its report - counts bool - }{ - {"a report left before the assignments", old, false}, - {"a report written since the assignments", placed.Add(30 * time.Second), true}, - } { - t.Run(c.name, func(t *testing.T) { - gate, set, said := start(t, true) - k, v := placing(t, "ace") // ace's key, first heard placing nothing - gate.Assignments(k, v, false, old.Add(-time.Hour)) - gate.Settled() - k, v = placing(t, "ace", NodeBuildModule) - if gate.Assignments(k, v, false, placed) { - t.Fatal("assignments with no report passed something on") - } - key, value := gateReport(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts { - t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) - } - // ace's assignments written again with another module beside: its standing does not move, nothing is said again. - k, v = placing(t, "ace", GoBuildModule, NodeBuildModule) - if gate.Assignments(k, v, false, placed.Add(time.Minute)) { - t.Fatal("the same assignments of mesh-build-node changed what was passed on") - } - if c.counts { - return - } - if v, none := agreedOf(set); v != "v26.12.1" || none != "" { - t.Fatalf("ace's left report is no builder's; got %q, %q", v, none) - } - if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want once why ace's report is left out", *said) - } - key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) - if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 { - t.Fatal("ace's new report was not passed on") - } - }) - } - - t.Run("no assignments held: every report counts", func(t *testing.T) { - gate, set, _ := start(t, false) - key, value := gateReport(t, "ace", "v26.12.0", old) - gate.Toolchain(NodeBuildModule, key, value, false) - gate.Settled() - if n := len(set.Entries("typescript")); n != 4 { - t.Fatalf("before the first assignments file every report counts; got %d", n) - } - k, v := placing(t, "g14", NodeBuildModule) - if !gate.Assignments(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 { - t.Fatalf("the first assignments leaves every node it does not place out; got %d", len(set.Entries("typescript"))) - } - if !gate.Assignments(k, nil, true, placed) || len(set.Entries("typescript")) != 4 { - t.Fatal("the last assignments gone: every report counts again") - } - }) - - // Design 47 *Each part acts on its own entries*: fed by the assignment half's revisions, the gate settles at the first, - // leaves out a node no revision places the build module on, and counts a node placed in a later revision only by a - // report written since that revision's time. - t.Run("fed by the assignment half", func(t *testing.T) { - set := &ToolchainSet{} - said := &[]string{} - gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) - for _, node := range append([]string{"ace"}, three...) { - key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute)) - if node == "ace" { - key, value = gateReport(t, node, "v26.12.0", old) - } - gate.Toolchain(NodeBuildModule, key, value, false) - } - revision := func(n int64, at time.Time, nodes ...string) meshwants.AssignmentRevision { + t.Run("the assignment half: the first revision settles, a later one moves the nodes", func(t *testing.T) { + gate := NewPlacedNodes(func(string, ...any) {}) + revision := func(n int64, nodes ...string) meshwants.AssignmentRevision { + at := old.Add(time.Duration(n) * time.Hour) r := meshwants.AssignmentRevision{Revision: n, Kind: "trunk", At: &at} for _, node := range nodes { r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: NodeBuildModule, Digest: "d-" + node}) } return r } - if !gate.Revision(revision(7, old.Add(-time.Hour), three...)) || len(set.Entries("typescript")) != 3 { - t.Fatalf("the first revision settles the gate and leaves ace out; %d entries", len(set.Entries("typescript"))) + if !gate.Revision(revision(7, three...)) || len(gate.Nodes()) != 3 { + t.Fatalf("the first revision: nodes %v", gate.Nodes()) } - if gate.Revision(revision(7, old.Add(-time.Hour), three...)) { - t.Fatal("the same revision again changed what was passed on") + if gate.Revision(revision(7, three...)) { + t.Fatal("the same revision again changed the counted nodes") } - if gate.Revision(revision(8, placed, "ace", "g14", "novox", "shanks")); len(set.Entries("typescript")) != 3 { - t.Fatal("ace's report from before revision 8 placed it counts") - } - if len(*said) < 2 || !strings.Contains((*said)[len(*said)-1], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") { - t.Fatalf("said %q, want why ace's old report is left out", *said) - } - key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) - if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 { - t.Fatal("ace's report written since revision 8 is not passed on") - } - if !gate.Revision(revision(9, placed.Add(time.Hour), "ace", "novox", "shanks")) || len(set.Entries("typescript")) != 3 || len(gate.Nodes()) != 3 { - t.Fatalf("g14 taken off in revision 9: %d entries, nodes %v", len(set.Entries("typescript")), gate.Nodes()) - } - }) - - t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) { - p := meshmoduleassignerclient.NewPlaced() - // ace's node key is the first revision heard of it: its module stands since before. - k, v := placing(t, "ace", NodeBuildModule) - p.Put(k, v, false, old) - if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 { - t.Errorf("a module key says which modules stand: %v", changed) - } - k2, v2 := placing(t, "g14") - p.Put(k2, v2, false, old) - k2, v2 = placing(t, "g14", NodeBuildModule) - if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule { - t.Errorf("a module placed says changed %v", changed) - } - if _, _, err := p.Put("novox.node", []byte("not json"), false, old); err == nil { - t.Error("a value that does not read was not said") - } - if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() { - t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok) - } - if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) { - t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok) - } - if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() { - t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode()) + if !gate.Revision(revision(9, "novox", "shanks")) || len(gate.Nodes()) != 2 { + t.Fatalf("g14 taken off: nodes %v", gate.Nodes()) } }) @@ -263,7 +84,7 @@ func TestToolchainGate(t *testing.T) { var logs bytes.Buffer logger := log.New(&logs, "", 0) refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_assignments") - gate, _, _ := start(t, false) + gate := NewPlacedNodes(logger.Printf) for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} { if _, err := FollowAssignments(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil { t.Errorf("a refused assignments stopped the start: %v", err) diff --git a/modules/mesh-delivery-builder/buildrun/toolchains.go b/modules/mesh-delivery-builder/buildrun/toolchains.go index a37200b9..ba2368c6 100644 --- a/modules/mesh-delivery-builder/buildrun/toolchains.go +++ b/modules/mesh-delivery-builder/buildrun/toolchains.go @@ -1,55 +1,54 @@ package buildrun import ( + "sort" "sync" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) -// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed). +// LanguageOf is the language each build module's toolchain reports hold. var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"} -// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches -// put them in. Safe for use from the watches and from a build at once. +// ToolchainsHeardState is the builder's old state section of the toolchains it heard (design 47 G §604, §546): one key +// ., the toolchain as the build seat said it. +const ( + ToolchainsHeardState = "toolchains-heard" + ToolchainsHeardBucket = "mesh-delivery-builder_" + ToolchainsHeardState +) + +// ToolchainSet holds each node's toolchain of a language, as the builder heard it (design 47 G §604). Safe for use from +// the consumer and from a build at once. type ToolchainSet struct { mu sync.Mutex by map[string]map[string]buildstep.Toolchain // language -> node -> entry } -// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key .) or its -// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going). -// A key or a value that is not an entry of the module's language is left out. -func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool { - language, ok := LanguageOf[module] - if !ok { - return false - } - node, keyLanguage, err := buildstep.ParseToolchainKey(key) - if err != nil || keyLanguage != language { - return false - } +// Set takes a node's toolchain in and answers whether the language's entries changed (a node's version, recipe or +// digest, or a node appearing). +func (s *ToolchainSet) Set(node string, tc buildstep.Toolchain) bool { s.mu.Lock() defer s.mu.Unlock() - if deleted { - if _, had := s.by[language][node]; !had { - return false - } - delete(s.by[language], node) - return true - } - tc, err := buildstep.DecodeToolchain(value) - if err != nil || tc.Language != language { - return false - } if s.by == nil { s.by = map[string]map[string]buildstep.Toolchain{} } - if s.by[language] == nil { - s.by[language] = map[string]buildstep.Toolchain{} + if s.by[tc.Language] == nil { + s.by[tc.Language] = map[string]buildstep.Toolchain{} } - cur, had := s.by[language][node] - s.by[language][node] = tc - return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe + cur, had := s.by[tc.Language][node] + s.by[tc.Language][node] = tc + return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe || cur.Digest != tc.Digest +} + +// Delete takes a node's toolchain of a language out and answers whether it was held. +func (s *ToolchainSet) Delete(node, language string) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, had := s.by[language][node]; !had { + return false + } + delete(s.by[language], node) + return true } // Entries are the nodes' entries of a language, in no particular order. @@ -62,3 +61,33 @@ func (s *ToolchainSet) Entries(language string) []buildstep.NodeToolchain { } return out } + +// AgreedDigest is a digest of the agreed version and the nodes reporting it. +type AgreedDigest struct { + Digest string + Nodes []string +} + +// Digests are the digests the nodes reporting the agreed toolchain's version report, sorted by digest, each with its +// nodes sorted (design 47 G §472, §574). A node that reports no digest is left out. +func Digests(entries []buildstep.NodeToolchain, agreed buildstep.Toolchain) []AgreedDigest { + by := map[string][]string{} + for _, e := range entries { + if e.Version == agreed.Version && e.Recipe == agreed.Recipe && e.Digest != "" { + by[e.Digest] = append(by[e.Digest], e.Node) + } + } + out := make([]AgreedDigest, 0, len(by)) + for d, nodes := range by { + sort.Strings(nodes) + out = append(out, AgreedDigest{Digest: d, Nodes: nodes}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Digest < out[j].Digest }) + return out +} + +// The builder's tools over the toolchains it heard (design 47 G §604, §546). +const ( + ToolchainsTool = "builder_toolchains" + ToolchainsDropTool = "builder_toolchains_drop" +) diff --git a/modules/mesh-delivery-builder/buildrun/toolchains_test.go b/modules/mesh-delivery-builder/buildrun/toolchains_test.go index 72aeab92..82b78c5e 100644 --- a/modules/mesh-delivery-builder/buildrun/toolchains_test.go +++ b/modules/mesh-delivery-builder/buildrun/toolchains_test.go @@ -1,50 +1,49 @@ package buildrun import ( + "reflect" "testing" - "time" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) -func tcValue(t *testing.T, language, version string) []byte { - t.Helper() - now := time.Now() - b, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: language, Version: version, Recipe: "1", At: now}) - if err != nil { - t.Fatal(err) - } - return b +func tc(language, version, digest string) buildstep.Toolchain { + return buildstep.Toolchain{Language: language, Version: version, Recipe: "1", Digest: digest} } -func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6 +func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6, design 47 G §604 var s ToolchainSet - if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) { + if !s.Set("novox", tc("go", "go1.27.1", "sha256:a")) { t.Error("a first entry did not count as a change") } - if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) { + if s.Set("novox", tc("go", "go1.27.1", "sha256:a")) { t.Error("the same entry again counted as a change") } - s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false) - s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false) + if !s.Set("novox", tc("go", "go1.27.1", "sha256:b")) { + t.Error("a new digest at one version did not count as a change") + } + s.Set("ace", tc("go", "go1.27.1", "sha256:b")) + s.Set("ace", tc("typescript", "tsc5", "")) if got := s.Entries("go"); len(got) != 2 { t.Errorf("go entries = %+v, want two", got) } if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" { t.Errorf("typescript entries = %+v", got) } - if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 { - t.Error("a delete did not remove the node's entry") + if !s.Delete("ace", "go") || len(s.Entries("go")) != 1 || s.Delete("ace", "go") { + t.Error("a delete did not remove the node's entry once") } - // What is not a toolchain of the module's language is left out. - s = ToolchainSet{} - if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) { - t.Error("a typescript toolchain in the go build module's bucket counted") - } - if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) { - t.Error("garbage or an unknown module counted") - } - if len(s.Entries("go")) != 0 { - t.Error("an entry was kept") +} + +// Design 47 G §472, §574: the digests are those the nodes reporting the agreed version report, sorted, with their nodes. +func TestDigestsOfTheAgreedVersion(t *testing.T) { + entries := []buildstep.NodeToolchain{ + {Node: "novox", Toolchain: tc("go", "1.26.0", "sha256:b")}, {Node: "ace", Toolchain: tc("go", "1.26.0", "sha256:a")}, + {Node: "g14", Toolchain: tc("go", "1.26.0", "sha256:a")}, {Node: "shanks", Toolchain: tc("go", "1.25.0", "sha256:c")}, + {Node: "old", Toolchain: tc("go", "1.26.0", "")}} + got := Digests(entries, tc("go", "1.26.0", "")) + want := []AgreedDigest{{Digest: "sha256:a", Nodes: []string{"ace", "g14"}}, {Digest: "sha256:b", Nodes: []string{"novox"}}} + if !reflect.DeepEqual(got, want) { + t.Errorf("digests = %+v, want %+v", got, want) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index bbd75eaf..7e201f0a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -372,6 +372,7 @@ func main() { }} tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) // The builder's expiring keys (state bounds): what each bounds and when it ends. tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription, Input: map[string]any{"type": "object", "properties": map[string]any{}}, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains.go new file mode 100644 index 00000000..06ec72b9 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains.go @@ -0,0 +1,82 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// heardState is the builder's state toolchains-heard as the tools bundle reads and deletes in it. +type heardState interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) + Delete(key string) error +} + +// toolchainsTools are builder_toolchains, which lists the toolchains the builder heard from the build seats, and +// builder_toolchains_drop, which deletes one key and writes it in the hand-act log (design 47 G §604, §546). The drop is +// of class act, with the default callers: the bus refuses another caller. +func toolchainsTools(st heardState, acts meshhandactsclient.Recorder, now func() time.Time) []stdio.Tool { + return []stdio.Tool{ + {Name: buildrun.ToolchainsTool, + Description: "The toolchains the builder heard from the build seats' toolchain-reported, one key . each: its version, " + + "recipe, the digest of its toolchain program and when it was said. The builder agrees a language's version from these, and forgets a node's " + + "when the node-engine seat says the build module stopped there. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + keys, err := st.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + rows := []map[string]any{} + for _, k := range keys { + e, err := st.Get(k) + if err != nil || e == nil { + continue + } + row := map[string]any{"key": k} + if tc, err := buildstep.DecodeToolchain(e.Value); err != nil { + row["error"] = err.Error() + } else { + row["version"], row["recipe"], row["digest"], row["said"] = tc.Version, tc.Recipe, tc.Digest, tc.At + } + rows = append(rows, row) + } + return map[string]any{"toolchains": rows}, nil + }}, + {Name: buildrun.ToolchainsDropTool, + Description: "Drop one node's toolchain of a language from what the builder heard, by hand: a build module that left a node and said nothing. " + + "The builder agrees without it; the node's build module says it again at its start and on each change. Written in the hand-act log (mesh-hand-acts).", + Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ + "key": map[string]any{"type": "string", "description": "., as builder_toolchains lists it"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + key, why := str("key"), str("why") + if _, _, err := buildstep.ParseToolchainKey(key); err != nil { + return nil, err + } + if why == "" { + return nil, fmt.Errorf("why is required") + } + if e, err := st.Get(key); err != nil || e == nil { + return nil, fmt.Errorf("%s holds no key %s", buildrun.ToolchainsHeardState, key) + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("dropped the toolchain %s from what the builder heard", key), + Why: why, Cause: buildrun.ToolchainsDropTool, Verb: "toolchains-drop"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err) + } + if err := st.Delete(key); err != nil { + return nil, err + } + return map[string]any{"key": key, "dropped": true, "at": now().UTC()}, nil + }}, + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains_test.go new file mode 100644 index 00000000..70429914 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/toolchains_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +type fakeHeard struct{ m map[string]json.RawMessage } + +func (f *fakeHeard) Keys() ([]string, error) { + var out []string + for k := range f.m { + out = append(out, k) + } + return out, nil +} +func (f *fakeHeard) Get(k string) (*stdio.StateEntry, error) { + v, ok := f.m[k] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: k, Value: v}, nil +} +func (f *fakeHeard) Delete(k string) error { delete(f.m, k); return nil } + +// Design 47 G §604, §546: builder_toolchains lists the keys; builder_toolchains_drop deletes one with its why, after the +// hand act is recorded, and nothing is deleted when it is not. +func TestToolchainsToolsListAndDropAKeyAfterTheHandAct(t *testing.T) { + data, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: "go", Version: "1.26.0", Recipe: "1", Digest: "sha256:a", At: time.Now()}) + if err != nil { + t.Fatal(err) + } + st := &fakeHeard{m: map[string]json.RawMessage{"ace.go": data}} + asked := 0 + var failAct error + acts := meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { + asked++ + return json.RawMessage(`{"id":"h1"}`), failAct + }} + tools := toolchainsTools(st, acts, time.Now) + run := func(name string, args map[string]any) (any, error) { + for _, tl := range tools { + if tl.Name == name { + return tl.Run(args) + } + } + t.Fatalf("no tool %s", name) + return nil, nil + } + got, err := run(buildrun.ToolchainsTool, nil) + rows := got.(map[string]any)["toolchains"].([]map[string]any) + if err != nil || len(rows) != 1 || rows[0]["key"] != "ace.go" || rows[0]["digest"] != "sha256:a" { + t.Fatalf("listed %v, %v", got, err) + } + if _, err := run(buildrun.ToolchainsDropTool, map[string]any{"key": "ace.go", "why": ""}); err == nil { + t.Fatal("a drop with no why was taken") + } + failAct = json.Unmarshal([]byte("x"), new(int)) + if _, err := run(buildrun.ToolchainsDropTool, map[string]any{"key": "ace.go", "why": "ace left"}); err == nil || !strings.Contains(err.Error(), "nothing was dropped") || len(st.m) != 1 { + t.Fatalf("a hand act not recorded still dropped: %v", err) + } + failAct = nil + if _, err := run(buildrun.ToolchainsDropTool, map[string]any{"key": "ace.go", "why": "ace left"}); err != nil || len(st.m) != 0 || asked < 2 { + t.Fatalf("the drop: %v, keys left %d", err, len(st.m)) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go index ec0783ec..4c50f3ca 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go @@ -69,7 +69,7 @@ func eventsConsumer(t *testing.T, b *bed, node string) { t.Fatal(err) } if _, err := b.js.CreateOrUpdateConsumer(b.ctx, eventsStream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder", - AckPolicy: jetstream.AckExplicitPolicy, FilterSubjects: []string{sweep.SubjectArtifactsGone}}); err != nil { + AckPolicy: jetstream.AckExplicitPolicy, FilterSubjects: []string{sweep.SubjectArtifactsGone, "mesh.seat.node-build-go.event.toolchain-reported.*", "mesh.seat.node-build-node.event.toolchain-reported.*"}}); err != nil { t.Fatal(err) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index c277bc6c..fbfc0f50 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -213,7 +213,7 @@ type bed struct { candidates jetstream.KeyValue source jetstream.KeyValue builds jetstream.KeyValue - tools map[string]jetstream.KeyValue + heard jetstream.KeyValue // the state toolchains-heard instances jetstream.KeyValue packages jetstream.KeyValue // the indexer's packages, served by its read tool and views family out *saidLog // the builder's process log, every instance's @@ -265,21 +265,21 @@ func startBedWith(t *testing.T, closed bool) *bed { } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) - b := &bed{ctx: ctx, js: js, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL(), dir: dir} - for module, stream := range toolchainStreams { - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil { + b := &bed{ctx: ctx, js: js, srvURL: srv.ClientURL(), dir: dir} + // EVENTS, where the build seats say toolchain-reported (design 47 G §492), with each builder's durable consumer of it, and + // the builder's state toolchains-heard (§604). + if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: eventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + t.Fatal(err) + } + for _, node := range []string{"n1", "n2"} { + if _, err := js.CreateOrUpdateConsumer(ctx, eventsStream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } - for _, node := range []string{"n1", "n2"} { - if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { - t.Fatal(err) - } - } - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buildrun.ToolchainsBucket(module), History: 1}) - if err != nil { - t.Fatal(err) - } - b.tools[module] = kv + } + if kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buildrun.ToolchainsHeardBucket, History: 1}); err != nil { + t.Fatal(err) + } else { + b.heard = kv } mk := func(name string, history int) jetstream.KeyValue { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) @@ -344,18 +344,15 @@ func (b *bed) logged(t *testing.T, module string) []builds.BuildLogEntry { return out } -// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does. +// report says a node's toolchain of a language on the build module's seat, as the seat's holder does (design 47 G §492). func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { t.Helper() - data, err := sdkstep.EncodeToolchain(sdkstep.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()}) + seat := map[string]string{"mesh-build-go": "node-build-go", "mesh-build-node": "node-build-node"}[module] + data, err := json.Marshal(map[string]string{"node": node, "language": language, "version": version, "recipe": recipe, "digest": "sha256:" + strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } - key, _ := sdkstep.ToolchainKey(node, language) - if _, err := b.tools[module].Put(b.ctx, key, data); err != nil { - t.Fatal(err) - } - if _, err := b.js.Publish(b.ctx, "mesh.stream."+module+".event.toolchain.reported", []byte(`{}`)); err != nil { + if _, err := b.js.Publish(b.ctx, "mesh.seat."+seat+".event.toolchain-reported."+node, data); err != nil { t.Fatal(err) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index c04e2884..9a743179 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -31,6 +31,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -247,12 +248,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } }() } - // The toolchains first, so a build that starts from the first modules value finds the nodes. - watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger) - // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the - // assignments is settled first, so the toolchains settle through the gate already knowing it. - gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) - watch.put = gate.Toolchain + // The toolchains the build seats say (design 47 G §492, §604): kept in the state toolchains-heard, read back at the start, so + // a build that starts from the first modules value finds the nodes; the assignments only counts the nodes (ADR 0624 §1). + gate := buildrun.NewPlacedNodes(logger.Printf) + heardTools := newToolchainsHeard(tools, b.NodesChanged, func(ctx context.Context, e meshbuilds.ToolchainAgreed) error { + return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectToolchainAgreed, "toolchain-agreed-"+e.Language+"-"+e.Version, e) + }, logger) + process.AwaitBucket(ctx, js, conn, buildrun.ToolchainsHeardBucket, "the toolchains heard are kept in memory only, and builder_toolchains shows none", logger.Printf). + Ready(func(kv jetstream.KeyValue) { + heardTools.bind(ctx, kvHeard{kvKeeper{ctx: ctx, kv: kv}}) + heardTools.readRunning(ctx, js) + dropped := func(e jetstream.KeyValueEntry) { + if !opOf(e) { + heardTools.Dropped(ctx, e.Key()) + } + } + if w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()); err != nil { + process.NotWatched(logger.Printf, buildrun.ToolchainsHeardBucket, err, "builder_toolchains_drop is not heard until the builder starts again") + } else { + go func() { errs <- follow(ctx, w, dropped) }() + } + }) // What each counted node's engine reads (ADR 0618 §2, ADR 0624 §1): asked through the node-engine seat's verb as the // builder's own account, for every node the assignments names, and kept in the state engine-reads once it is bound. reads := newEngineReads(calls.CallKeptOn, gate, logger.Printf) @@ -284,7 +300,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // asked when the bound ends), is call-unanswered in the builder's findings (issue 1966, ADR 0668 §3, ADR 0596 §2). reads.callsOf(found, pubBoundsKV.KV, heardBy(calls.CallKept), logger.Printf) b.Forms = reads.built - // Until the assignments's first view is answered the gate passes every report on, as before the assignments is settled. + // The assignments names the nodes the builder counts, once its first view is answered. assignmentsErrs := make(chan error, 1) go func() { select { @@ -299,12 +315,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } assignmentsErrs <- <-pe }() - if err := watch.start(); err != nil { - return err - } - // A build module says each time it wrote its toolchains key: its stream, and the builder's durable consumer on it, are - // made by the controller. Where the stream is not there yet the builder says so and is started again with the build - // module. // Each consumer that is not there yet is the finding family-not-heard naming its stream until it binds at a later // start (heard.Writer; ADR 0615 §5); one there that cannot be heard stops the builder, as before. notHeard := found.notHeard(ctx) @@ -324,9 +334,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil }) } - for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} { - hear(toolchainStreams[module], module+"'s toolchain reports are not heard until the builder starts again", func(m jetstream.Msg) { watch.Reported(m) }) - } // The seat mesh-artifact-sweep says artifacts-gone when it found a key's blob gone (issue 1937, ADR 0571 §2): the // builder's durable consumer on EVENTS, made by the grants part from the manifest's consumes. Where it is not there yet the builder says // so; it still asks held at its start and at every module value. @@ -336,7 +343,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} events := func(m jetstream.Msg) { - if reads.heard(m, logger.Printf) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) + if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } heardGone(ctx, b, m, logger.Printf) @@ -430,8 +437,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-assignmentsErrs: return err - case err := <-watch.errs: - return err } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go index 6b3b86c3..2c1549b5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go @@ -26,7 +26,7 @@ func TestTheNamesAreTheManifests(t *testing.T) { } for _, read := range []string{ gateModule + ".source", - "mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", } { if !slices.Contains(m.Reads, read) { t.Errorf("the builder reads %s, its manifest's reads are %v", read, m.Reads) @@ -40,7 +40,7 @@ func TestTheNamesAreTheManifests(t *testing.T) { } for want, got := range map[string]string{ "mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), - "mesh-build-node_toolchains": buildrun.ToolchainsBucket("mesh-build-node"), "mesh-delivery-indexer_packages": packagesBucket, + buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-indexer_packages": packagesBucket, "mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket, } { if want != got { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go index 1cc99503..cd9a1702 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go @@ -28,7 +28,7 @@ const engineReadsBucket = builds.BuilderModule + "_" + EngineReadsState // engineReads keeps the statements and the counted nodes for the builder's forms (build.Builder.Forms). type engineReads struct { keeper *enginereads.Keeper - gate *buildrun.ToolchainGate + gate *buildrun.PlacedNodes mu sync.Mutex counted []string // window is the old reference form's window over these statements (DLV-B9); nil keeps none. @@ -39,7 +39,7 @@ type engineReads struct { } // newEngineReads asks through call (process.Conn.CallKeptOn, a kept call; ADR 0668) and keeps each answer in the state once it is bound. -func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, logf func(string, ...any)) *engineReads { +func newEngineReads(call enginereads.Caller, gate *buildrun.PlacedNodes, logf func(string, ...any)) *engineReads { r := &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf}} r.keeper.Changed = func() { r.stepWindow() diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index d3d4bf22..cafc3e8a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -34,24 +34,16 @@ import ( "mesh-delivery-builder/internal/versions" ) -// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the -// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and -// its grant. The gate over the seat's assignments, and its refusal, are mesh-sdk's (buildrun.ToolchainGate, +// 1945 (a part exits at start for a bucket it works without): a refused bucket it cannot work without is an error naming the bucket and +// its grant. The gate over the seat's assignments, and its refusal, are mesh-sdk's (buildrun.PlacedNodes, // issue 774). func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard) t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding) t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard) - set := &buildrun.ToolchainSet{} var logs bytes.Buffer logger := log.New(&logs, "", 0) - refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")} - if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { - t.Errorf("a refused toolchains bucket stopped the start: %v", err) - } - if !strings.Contains(logs.String(), "cannot be read") { - t.Errorf("the refusals were not said: %q", logs.String()) - } + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")} if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) @@ -74,7 +66,7 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { // away node keeps its last statement; a node the assignments no longer names leaves the count, its key removed; and // builder_engine_reads drops one node's key. func theCountedNodesEngineReads(t *testing.T) { - gate := buildrun.NewToolchainGate(nil, func(string, string, []byte, bool) bool { return false }, t.Logf) + gate := buildrun.NewPlacedNodes(t.Logf) place := func(nodes ...string) { for _, node := range []string{"n1", "n2"} { pl := meshmoduleassigner.Assignments{Node: node} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 82734a16..62c88a80 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -2,107 +2,290 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "log" "strings" "sync" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" - "git.novox.be/novox/mesh-sdk/go/process" + "time" "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) -// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it -// (declared by the build modules, heard by this module through consumes). -var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO", buildrun.NodeBuildModule: "TOOLCHAINS_NODE"} - -// toolchainWatch keeps the build modules' toolchains in a buildrun.ToolchainSet and tells the builder when one changed. A -// build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437 -// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key -// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched. -type toolchainWatch struct { - ctx context.Context - js bucketOpener - set *buildrun.ToolchainSet - // put takes a change of a toolchains bucket in: set.Put, or the gate of the assigned build nodes before it (issue 1940). - put func(module, key string, value []byte, deleted bool) bool - changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again - logger *log.Logger - errs chan error - mu sync.Mutex - watched map[string]bool +// toolchainReportedPrefixes are the subjects the three build seats say toolchain-reported on, a node its last token (design +// 47 G §492). The builder hears them through the consumer of its manifest's consumes, and keeps what it heard in its +// state toolchains-heard (§604). +var toolchainReportedPrefixes = []string{ + strings.TrimSuffix(nodebuildgo.SubjectToolchainReportedAll, "*"), + strings.TrimSuffix(nodebuildnode.SubjectToolchainReportedAll, "*"), } -func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch { - return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}} +// heardKept is the state toolchains-heard: one key . (design 47 G §604). +type heardKept interface { + Put(key string, value []byte) error + Delete(key string) error } -// start watches the buckets that exist now; a missing one is said and waits for its build module's event. -func (w *toolchainWatch) start() error { - for module := range buildrun.LanguageOf { - if err := w.ensure(module); err != nil { - return err +// heardState is the state as the builder reads it back at its start. +type heardState interface { + heardKept + Keys() ([]string, error) + Get(key string) ([]byte, error) +} + +// kvHeard is heardState over the bound bucket. +type kvHeard struct { + kvKeeper +} + +func (k kvHeard) Keys() ([]string, error) { + keys, err := k.kv.Keys(k.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func (k kvHeard) Get(key string) ([]byte, error) { + e, err := k.kv.Get(k.ctx, key) + if err != nil { + return nil, err + } + return e.Value(), nil +} + +// toolchainsHeard keeps the toolchains the build seats said (design 47 G §492, §604), forgets a node's on the node-engine +// seat's stopped (§546), and says toolchain-agreed with its digests when a language's agreed version changes (§472, §574). +type toolchainsHeard struct { + set *buildrun.ToolchainSet + changed func(context.Context) + say func(ctx context.Context, e meshbuilds.ToolchainAgreed) error + now func() time.Time + logf func(string, ...any) + + mu sync.Mutex + kept heardKept // nil until the state is bound: what was heard is kept in memory only, and written then + agreed map[string]string // language -> the version last said + runAt map[string]time.Time // . -> the `at` of the newest running read +} + +func newToolchainsHeard(set *buildrun.ToolchainSet, changed func(context.Context), say func(context.Context, meshbuilds.ToolchainAgreed) error, logger *log.Logger) *toolchainsHeard { + return &toolchainsHeard{set: set, changed: changed, say: say, now: time.Now, logf: logger.Printf, agreed: map[string]string{}, runAt: map[string]time.Time{}} +} + +// bind takes the bound state in: the toolchains it holds are read back (a restart needs no node to report again), and +// what was heard before it was bound is written. +func (h *toolchainsHeard) bind(ctx context.Context, state heardState) { + keys, err := state.Keys() + if err != nil { + h.logf("%s cannot be read, so the toolchains of before this start are not known: %v", buildrun.ToolchainsHeardState, err) + } + for _, k := range keys { + value, err := state.Get(k) + if err != nil { + continue + } + node, language, err := buildstep.ParseToolchainKey(k) + tc, derr := buildstep.DecodeToolchain(value) + if err != nil || derr != nil || tc.Language != language { + h.logf("%s.%s holds what is not a toolchain, and is left: %v %v", buildrun.ToolchainsHeardState, k, err, derr) + continue + } + h.set.Set(node, tc) + } + h.mu.Lock() + h.kept = state + h.mu.Unlock() + for _, language := range buildrun.LanguageOf { + for _, e := range h.set.Entries(language) { + h.keep(e.Node, e.Toolchain) } } - return nil + h.agree(ctx) + h.changed(ctx) } -func (w *toolchainWatch) ensure(module string) error { - w.mu.Lock() - defer w.mu.Unlock() - if w.watched[module] { - return nil +func (h *toolchainsHeard) keep(node string, tc buildstep.Toolchain) { + h.mu.Lock() + kept := h.kept + h.mu.Unlock() + if kept == nil { + return } - name := buildrun.ToolchainsBucket(module) - kv, err := w.js.KeyValue(w.ctx, name) - switch { - case errors.Is(err, jetstream.ErrBucketNotFound): - w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) - return nil - case err != nil: - // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. - w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) - return nil - } - watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) + key, err := buildstep.ToolchainKey(node, tc.Language) if err != nil { - process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) - return nil - } - w.watched[module] = true - go func() { - w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { - if w.put(module, e.Key(), e.Value(), !opOf(e)) { - w.changed(w.ctx) - } - }) - }() - w.changed(w.ctx) // what the settle read is new to the builder - return nil -} - -// delivered is the part of a JetStream message the watch uses. -type delivered interface { - Subject() string - Ack() error - Term() error -} - -// Reported takes one toolchain.reported event of a build module in: the module is read off the subject, -// mesh.stream..event.toolchain.reported, and its bucket is watched if it is not. A subject of no build module is -// terminated; a failure stays unacknowledged, and the server delivers it again. -func (w *toolchainWatch) Reported(m delivered) { - module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.") - if _, known := buildrun.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") { - w.logger.Printf("%s: not a build module's event, dropped", m.Subject()) - _ = m.Term() return } - if err := w.ensure(module); err != nil { - w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) - return + data, err := buildstep.EncodeToolchain(tc) + if err == nil { + err = kept.Put(key, data) + } + if err != nil { + h.logf("%s.%s could not be written, so it is kept in memory only: %v", buildrun.ToolchainsHeardState, key, err) + } +} + +// Reported takes one toolchain-reported of a build seat in; it answers whether the message was one. The toolchain is +// kept in the state, and builds that waited for the nodes to agree are looked at again. +func (h *toolchainsHeard) Reported(ctx context.Context, m jetstream.Msg) bool { + subject := m.Subject() + var prefix string + for _, p := range toolchainReportedPrefixes { + if strings.HasPrefix(subject, p) { + prefix = p + } + } + if prefix == "" { + return false } _ = m.Ack() + var e nodebuildgo.ToolchainReported + if err := json.Unmarshal(m.Data(), &e); err != nil || e.Node == "" || e.Node != strings.TrimPrefix(subject, prefix) { + h.logf("%s holds no toolchain of its node, and is dropped: %v", subject, err) + return true + } + tc := buildstep.Toolchain{Language: e.Language, Version: e.Version, Recipe: e.Recipe, Digest: e.Digest, At: h.now().UTC()} + if _, err := buildstep.EncodeToolchain(tc); err != nil { + h.logf("%s is no toolchain, and is dropped: %v", subject, err) + return true + } + if h.set.Set(e.Node, tc) { + h.keep(e.Node, tc) + h.agree(ctx) + h.changed(ctx) + } + return true +} + +// Running takes one running of the node-engine seat in (design 47 G §546); it answers whether the message was one. A +// running with no fingerprint is stopped (§365): the key . of the build module of that language on that +// node is deleted. An event older by `at` than one taken before it changes nothing. +func (h *toolchainsHeard) Running(ctx context.Context, m jetstream.Msg) bool { + if !strings.HasPrefix(m.Subject(), strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) { + return false + } + _ = m.Ack() + h.running(ctx, m.Data()) + return true +} + +func (h *toolchainsHeard) running(ctx context.Context, data []byte) { + var r nodeengine.Running + if err := json.Unmarshal(data, &r); err != nil { + h.logf("a running that does not read is dropped: %v", err) + return + } + id := r.Node + "." + r.Module + h.mu.Lock() + if prev, ok := h.runAt[id]; ok && r.At.Before(prev) { + h.mu.Unlock() + return + } + h.runAt[id] = r.At + kept := h.kept + h.mu.Unlock() + language, build := buildrun.LanguageOf[r.Module] + if !build || r.Fingerprint != "" { + return + } + if !h.set.Delete(r.Node, language) { + return + } + key, _ := buildstep.ToolchainKey(r.Node, language) + if kept != nil { + if err := kept.Delete(key); err != nil { + h.logf("%s.%s could not be deleted: %v", buildrun.ToolchainsHeardState, key, err) + } + } + h.agree(ctx) + h.changed(ctx) +} + +// Dropped is the key's drop by a person (builder_toolchains_drop), taken from the state's watch. +func (h *toolchainsHeard) Dropped(ctx context.Context, key string) { + node, language, err := buildstep.ParseToolchainKey(key) + if err != nil { + return + } + if h.set.Delete(node, language) { + h.agree(ctx) + h.changed(ctx) + } +} + +// agree says toolchain-agreed for each language whose agreed version is not the one last said (design 47 G §472: said +// only when the agreed version changes, ADR 0584 §1). +func (h *toolchainsHeard) agree(ctx context.Context) { + for _, language := range buildrun.LanguageOf { + entries := h.set.Entries(language) + tc, none := buildstep.AgreedToolchain(entries) + if none != "" { + continue + } + h.mu.Lock() + same := h.agreed[language] == tc.Version + if !same { + h.agreed[language] = tc.Version + } + h.mu.Unlock() + if same { + continue + } + at := h.now().UTC() + e := meshbuilds.ToolchainAgreed{V: 1, At: &at, Language: language, Version: tc.Version, Digests: []meshbuilds.ToolchainAgreedDigestsItem{}} + for _, d := range buildrun.Digests(entries, tc) { + e.Digests = append(e.Digests, meshbuilds.ToolchainAgreedDigestsItem{Digest: d.Digest, Nodes: d.Nodes}) + } + if err := h.say(ctx, e); err != nil { + h.logf("saying %s of %s %s: %v", meshbuilds.EventToolchainAgreed, language, tc.Version, err) + h.mu.Lock() + delete(h.agreed, language) // said again at the next change + h.mu.Unlock() + } + } +} + +// lastPerSubject reads the last message of every subject of a stream kept last per subject, through an ordered consumer +// made for that read (design 47 G §556), and hands each to each; it answers when the last of them was handed. +func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte)) error { + cons, err := js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) + if err != nil { + return fmt.Errorf("reading %s: %w", stream, err) + } + info, err := cons.Info(ctx) + if err != nil { + return fmt.Errorf("reading %s: %w", stream, err) + } + pending := int(info.NumPending) + if pending == 0 { + return nil + } + msgs, err := cons.Messages() + if err != nil { + return fmt.Errorf("reading %s: %w", stream, err) + } + defer msgs.Stop() + for ; pending > 0; pending-- { + m, err := msgs.Next() + if err != nil { + return fmt.Errorf("reading %s: %w", stream, err) + } + each(m.Subject(), m.Data()) + } + return nil +} + +// readRunning is the start's read of every node's module's last running (§556), before the consumer's. +func (h *toolchainsHeard) readRunning(ctx context.Context, js jetstream.JetStream) { + if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte) { h.running(ctx, data) }); err != nil { + h.logf("the running of the node-engine seat is not read at the start, so a build module stopped while this builder was away is forgotten at its next running: %v", err) + } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go new file mode 100644 index 00000000..8018670c --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "log" + "sort" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" +) + +// memState is the state toolchains-heard in memory. +type memState struct { + mu sync.Mutex + m map[string][]byte +} + +func (s *memState) Put(k string, v []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.m == nil { + s.m = map[string][]byte{} + } + s.m[k] = v + return nil +} +func (s *memState) Delete(k string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.m, k) + return nil +} +func (s *memState) Get(k string) ([]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.m[k], nil +} +func (s *memState) Keys() ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for k := range s.m { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// msg is a delivered message as the builder's consumer hands it. +type msg struct { + jetstream.Msg + subject string + data []byte +} + +func (m msg) Subject() string { return m.subject } +func (m msg) Data() []byte { return m.data } +func (m msg) Ack() error { return nil } + +type heardRig struct { + h *toolchainsHeard + state *memState + said []meshbuilds.ToolchainAgreed + changed int +} + +func newHeardRig(t *testing.T, state *memState) *heardRig { + t.Helper() + r := &heardRig{state: state} + r.h = newToolchainsHeard(&buildrun.ToolchainSet{}, func(context.Context) { r.changed++ }, + func(_ context.Context, e meshbuilds.ToolchainAgreed) error { r.said = append(r.said, e); return nil }, log.New(io.Discard, "", 0)) + r.h.bind(context.Background(), state) + return r +} + +func (r *heardRig) report(t *testing.T, seat, node, language, version, digest string) { + t.Helper() + data, _ := json.Marshal(map[string]string{"node": node, "language": language, "version": version, "recipe": "1", "digest": digest}) + if !r.h.Reported(context.Background(), msg{subject: "mesh.seat." + seat + ".event.toolchain-reported." + node, data: data}) { + t.Fatal("a toolchain-reported was not taken") + } +} + +func (r *heardRig) running(t *testing.T, node, module, fingerprint string, at time.Time) { + t.Helper() + data, _ := json.Marshal(nodeengine.Running{Node: node, Module: module, Fingerprint: fingerprint, At: at}) + if !r.h.Running(context.Background(), msg{subject: nodeengine.SubjectRunning(node, module), data: data}) { + t.Fatal("a running was not taken") + } +} + +// Design 47 G §492, §604, §472, §574, §546: the builder agrees from the toolchain-reported it heard, keeps them in +// toolchains-heard, says toolchain-agreed with the digests only when the agreed version changes, and forgets a node's on +// the node-engine seat's stopped. +func TestTheBuilderKeepsTheToolchainsItHeardAndForgetsThemOnStopped(t *testing.T) { + da, db := "sha256:a", "sha256:b" + state := &memState{} + r := newHeardRig(t, state) + r.report(t, "node-build-go", "ace", "go", "1.26.0", da) + r.report(t, "node-build-go", "g14", "go", "1.26.0", da) + r.report(t, "node-build-go", "novox", "go", "1.26.0", db) + keys, _ := state.Keys() + if len(keys) != 3 || keys[0] != "ace.go" { + t.Fatalf("the state holds %v, want the three nodes' keys .go", keys) + } + if len(r.said) != 1 || r.said[0].Version != "1.26.0" || r.said[0].Language != "go" { + t.Fatalf("toolchain-agreed said %+v, want one for go 1.26.0 (said only when the version changes)", r.said) + } + // The first report agreed it with one digest; a later digest of the same version is not said again. + if len(r.said[0].Digests) != 1 { + t.Fatalf("the first agreement carried %+v", r.said[0].Digests) + } + // A restarted builder reads the keys back and agrees the same version with no new report; the digests it says are all three nodes'. + again := newHeardRig(t, state) + if len(again.said) != 1 || again.said[0].Version != "1.26.0" || len(again.said[0].Digests) != 2 || again.said[0].Digests[0].Nodes[1] != "g14" { + t.Fatalf("a restarted builder said %+v, want go 1.26.0 with the digests of its three nodes", again.said) + } + // A version change is said; the stopped build module of ace takes ace's key out, a placement taking nothing. + again.report(t, "node-build-go", "ace", "go", "1.27.0", da) + again.report(t, "node-build-go", "g14", "go", "1.27.0", da) + if len(again.said) != 2 || again.said[1].Version != "1.27.0" { + t.Fatalf("a changed agreed version was not said: %+v", again.said) + } + at := time.Now() + again.running(t, "ace", buildrun.GoBuildModule, "src2:x", at) // running: nothing is forgotten + if k, _ := state.Keys(); len(k) != 3 { + t.Fatalf("a running forgot a key: %v", k) + } + again.running(t, "ace", buildrun.GoBuildModule, "", at.Add(time.Second)) // stopped: no fingerprint + if k, _ := state.Keys(); len(k) != 2 || k[0] != "g14.go" { + t.Fatalf("stopped on ace left %v, want g14.go and novox.go", k) + } + // An older running read after (the start's read of the stream kept last per subject) changes nothing. + before := again.changed + again.running(t, "g14", buildrun.GoBuildModule, "", at) + again.running(t, "g14", buildrun.GoBuildModule, "src2:y", at.Add(-time.Hour)) + if k, _ := state.Keys(); len(k) != 1 || k[0] != "novox.go" || again.changed != before+1 { + t.Fatalf("after g14 stopped and an older running: %v", k) + } + // A key dropped by a person leaves the set. + again.h.Dropped(context.Background(), "novox.go") + if len(again.h.set.Entries("go")) != 0 { + t.Fatal("a dropped key stayed in the set") + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index fbe610ac..ed1460ab 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -227,6 +227,25 @@ func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account return err } +// emitSeatEvent says an event of a seat the builder holds, on the seat's subject (mesh.seat..event.). +func emitSeatEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, subject, id string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + msg := nats.NewMsg(subject) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) + return err +} + // The events on the builder's consumer of EVENTS that wake a floor (design 51 *The floor*, DLV-B3b). var ( manifestAcceptedSubject = meshcatalog.SubjectManifestAccepted diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 3d333a4d..cf46a190 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -13,6 +13,8 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/internal/build/fakes_test.go b/modules/mesh-delivery-builder/internal/build/fakes_test.go index bd18bba5..fb490343 100644 --- a/modules/mesh-delivery-builder/internal/build/fakes_test.go +++ b/modules/mesh-delivery-builder/internal/build/fakes_test.go @@ -75,14 +75,9 @@ func presenceValue(t *testing.T, online bool) []byte { return b } -// tcValue is a toolchains value, as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain). -func tcValue(t *testing.T, language, version string) []byte { - t.Helper() - b, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now().UTC()}) - if err != nil { - t.Fatal(err) - } - return b +// tcOf is a node's toolchain of a language, as the builder keeps what it heard. +func tcOf(language, version string) buildstep.Toolchain { + return buildstep.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now().UTC()} } // tcs are the toolchain entries of a language on nodes, as the toolchains set gives them. diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 4d28e0a9..1c471d37 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -180,8 +180,8 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { t.Helper() r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} for _, n := range []string{"ace", "novox"} { - r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false) - r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false) + r.tools.Set(n, tcOf("go", "go1.27.1")) + r.tools.Set(n, tcOf("typescript", "tsc5")) } r.asks = &buildrun.FakeAsks{Answer: r.answer} r.b = &Builder{ @@ -235,8 +235,8 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 t.Run("ADR 0624: a done key keeps each form a counted node reads", aDoneKeyKeepsEachFormACountedNodeReads) r := newRig(t, goBundle, archiveA, tsBundle) // ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out. - r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false) - r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false) + r.tools.Set("g14", tcOf("typescript", "tsc5")) + r.tools.Set("ace", tcOf("typescript", "tsc4")) defer func() { if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") { t.Errorf("the key does not say ace sits out: %s", st) @@ -717,8 +717,8 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls)) } // The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds. - r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false) - r.tools.Put(buildrun.GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false) + r.tools.Set("ace", tcOf("go", "go1.27.2")) + r.tools.Set("novox", tcOf("go", "go1.27.2")) r.b.Reconsider(context.Background()) r.b.Wait() if len(r.st.vals) != 0 { @@ -740,11 +740,7 @@ func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) { calls := len(r.bm.calls) fp2 := "src2:" + hex64('2') for _, n := range []string{"ace", "novox"} { - b, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now().UTC()}) - if err != nil { - t.Fatal(err) - } - r.tools.Put(buildrun.GoBuildModule, n+".go", b, false) + r.tools.Set(n, buildstep.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now().UTC()}) } r.b.Reconsider(context.Background()) r.b.Wait() @@ -772,7 +768,7 @@ func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) { func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) - r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) + r.tools.Set("ace", tcOf("go", "go1.26.0")) r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 5c6c1068..0b5e7521 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -102,13 +102,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a // node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3). - if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", + if want := []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } @@ -137,18 +137,18 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } - // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it - // starts; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the + // The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no + // build module's toolchains; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the // controller's no longer (1126/B9, ADR 0590). // Design 51 *The floor*: the indexer's manifest-accepted and the condition keeper's condition-cleared wake a silent floor. // DLV-B4: the four registry seats' outcomes, each a seat event `.` on the shared events stream. // DLV-M1: the indexer's and the assigner's views families of the states it follows. if want := []string{"container-image-registry.kept", "container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed", "go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed", - "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", + "mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", "mesh-delivery-assigner.views.assignments", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", - "mesh-delivery-indexer.views.packages", "node-engine.manifest-reads-changed", + "mesh-delivery-indexer.views.packages", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) @@ -161,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f3d82244..3581b792 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -26,6 +26,8 @@ "builder_reserved_drop", "builder_seat_queue", "builder_stuck", + "builder_toolchains", + "builder_toolchains_drop", "builder_versions", "reserved", "builder_first_views", @@ -45,14 +47,15 @@ "go-package-registry.refused", "go-package-registry.removed", "mesh-artifact-sweep.artifacts-gone", - "mesh-build-go.toolchain.reported", - "mesh-build-node.toolchain.reported", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", "mesh-delivery-forge-bridge.forge.**", + "node-build-go.toolchain-reported", + "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", + "node-engine.running", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", @@ -103,6 +106,7 @@ { "name": "old-form", "history": 1 }, { "name": "bounds", "history": 1, "key-ttl": true }, { "name": "queues", "history": 1 }, + { "name": "toolchains-heard", "history": 1 }, { "name": "reserved", "history": 1 }, { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, @@ -113,10 +117,8 @@ ], "reads": [ "mesh-build-go.steps", - "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", - "mesh-build-node.toolchains", "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source" ], From b584af3f6191b917642c48d1aed81884cac44bb7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:53:19 +0100 Subject: [PATCH 9/9] BL14: the builder holds an entry whose fingerprint names another toolchain, in its state waits, with wait-held and wait-ended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An entry whose fingerprint names a toolchain version of its language other than the agreed one is held in the state waits (one key .), says wait-held and wait-ended (ran when the agreed toolchain names the version, released by builder_release) on its stream waits, says wait-held again for every key at its start; builder_waits and builder_release; its setting toolchain-wait-minutes. Not built: the bound's finding toolchain-waits (its scope is not in the design). Builds: design 47 G §672, §267, §276, §633 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-builder-tools/main.go | 9 ++ .../cmd/mesh-delivery-builder-tools/waits.go | 78 +++++++++++ .../mesh-delivery-builder-tools/waits_test.go | 46 +++++++ .../cmd/mesh-delivery-builder/main.go | 18 +++ .../cmd/mesh-delivery-builder/reasks.go | 23 ++++ .../cmd/mesh-delivery-builder/waits.go | 125 +++++++++++++++++ .../internal/build/run.go | 22 ++- .../internal/build/waits.go | 126 ++++++++++++++++++ .../internal/build/waits_test.go | 84 ++++++++++++ .../mesh-delivery-builder/manifest_test.go | 40 ++++-- modules/mesh-delivery-builder/module.json | 19 ++- 11 files changed, 574 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go create mode 100644 modules/mesh-delivery-builder/internal/build/waits.go create mode 100644 modules/mesh-delivery-builder/internal/build/waits_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 7e201f0a..dea457ae 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -372,6 +372,15 @@ func main() { }} tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + ws := stdio.State(build.WaitsState) + wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { + e, err := ws.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil + }} + tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) // The builder's expiring keys (state bounds): what each bounds and when it ends. tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits.go new file mode 100644 index 00000000..2185b9d7 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// waitsTools are builder_waits, which reads the builder's toolchain waits, and builder_release, which builds an entry with +// the running build module (design 47 G §276, §672): a hand act, recorded in the hand-act log first, then written to reasks +// for the builder to answer. +func waitsTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: build.WaitsTool, + Description: "The builder's toolchain waits: each entry held because its fingerprint names a toolchain version of its language other than the " + + "agreed one, with its module, fingerprint, what it waits on (toolchain:@) and since when. It ends when the agreed toolchain " + + "names that version (ran) or when " + build.ReleaseTool + " builds it with the running build module (released). Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + keys, err := rr.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + rows := []build.ToolchainWait{} + for _, k := range keys { + raw, ok, err := rr.Get(k) + if err != nil || !ok { + continue + } + var w build.ToolchainWait + if json.Unmarshal(raw, &w) == nil { + rows = append(rows, w) + } + } + return map[string]any{"waits": rows}, nil + }}, + {Name: build.ReleaseTool, + Description: "Build a held entry with the running build module now, whatever toolchain its fingerprint names (a toolchain wait of " + build.WaitsTool + + "), and end its wait released. Written in the hand-act log (mesh-hand-acts).", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name"}, + "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:, as builder_waits shows it"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewRelease(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("released the toolchain wait of %s at %s by hand", str(args, "module"), str(args, "fp")), + Why: str(args, "why"), Cause: build.ReleaseTool, Verb: "release"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was released: %w", err) + } + raw, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if a.Refused { + return nil, errors.New(a.Answer) + } + return map[string]any{"key": key, "answer": a.Answer}, nil + }}, + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go new file mode 100644 index 00000000..e782acf2 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// Design 47 G §276, §672: builder_waits lists the held entries; builder_release records the hand act first and asks the +// builder through reasks, whose refusal is the tool's error. +func TestWaitsToolsListTheWaitsAndReleaseAfterTheHandAct(t *testing.T) { + wait, _ := json.Marshal(build.ToolchainWait{Module: "m", Fingerprint: "src2:" + strings.Repeat("1", 64), Language: "go", Version: "go1.26.0", Since: time.Now()}) + rr := versions.Reader{Keys: func() ([]string, error) { return []string{"m.k"}, nil }, + Get: func(string) (json.RawMessage, bool, error) { return wait, true, nil }} + var asked []build.ReAsk + answer := func(key string, value []byte) (json.RawMessage, error) { + a, _ := build.DecodeReAsk(value) + asked = append(asked, a) + a.Answered, a.Answer = time.Now(), "released: built" + return json.Marshal(a) + } + recorded := 0 + acts := meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { recorded++; return json.RawMessage(`{}`), nil }} + tools := waitsTools(rr, answer, time.Now, acts) + byName := map[string]func(map[string]any) (any, error){} + for _, tl := range tools { + byName[tl.Name] = tl.Run + } + got, err := byName[build.WaitsTool](nil) + if rows := got.(map[string]any)["waits"].([]build.ToolchainWait); err != nil || len(rows) != 1 || rows[0].On() != "toolchain:go@go1.26.0" { + t.Fatalf("waits = %v, %v", got, err) + } + if _, err := byName[build.ReleaseTool](map[string]any{"module": "m", "fp": "src2:" + strings.Repeat("1", 64), "why": ""}); err == nil || recorded != 0 { + t.Fatalf("a release with no why was taken: %v, %d recorded", err, recorded) + } + if _, err := byName[build.ReleaseTool](map[string]any{"module": "m", "fp": "src2:" + strings.Repeat("1", 64), "why": "the build module runs"}); err != nil || + recorded != 1 || len(asked) != 1 || asked[0].Mode != build.ModeRelease || asked[0].Module != "m" { + t.Fatalf("release: %v, %d recorded, asked %+v", err, recorded, asked) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 9a743179..fa35f4ac 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -14,6 +14,7 @@ import ( "os" "os/signal" "slices" + "strconv" "strings" "sync" "syscall" @@ -48,6 +49,8 @@ const ( // build compiles): from its start, never from its put, so an ask waiting in its stream for a free slot is not failed // (buildrun.Asker.Bound; ADR 0525 §3, §6). defaultStepBound = 30 * time.Minute + // defaultToolchainWait is the bound of a toolchain wait, its setting toolchain-wait-minutes (design 47 G §672). + defaultToolchainWait = 60 * time.Minute ) func main() { @@ -183,6 +186,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf} b.Left, b.Ended = unput.left, unput.ended b.Numbers = build.VersionNumbers{N: numberer} + // The builder's toolchain waits (design 47 G §672): its state waits, and wait-held and wait-ended on its stream waits; its + // bound is its setting toolchain-wait-minutes. + waitBound := defaultToolchainWait + if v := strings.TrimSpace(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES")); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return fmt.Errorf("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES %q is not a number of minutes", v) + } + waitBound = time.Duration(n) * time.Minute + } + waitsBucket := process.AwaitBucket(ctx, js, conn, build.WaitsBucket, "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf) + b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error { + return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body) + }, waitBound, logger.Printf) + waitsBucket.Ready(func(kv jetstream.KeyValue) { b.Waits.(*waitsKept).start(ctx, kv) }) // The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled // and version-released heard; nothing is reserved before the fill. reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index bbcba722..26da5ebb 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -23,6 +23,7 @@ type askAgainer interface { AskAgain(key, why string) []string BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult + Release(ctx context.Context, module, fp, why, by string) string } // buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits @@ -53,6 +54,10 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream announcedAgain(ctx, b, kv, e, r, logf) return } + if r.Mode == build.ModeRelease { + released(ctx, b, kv, e, r, logf) + return + } if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { // A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch. go versionActed(ctx, acts, kv, e, r, logf) @@ -191,3 +196,21 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) } } + +// released answers a reasks value of mode release (the tool builder_release, design 47 G §276, §672): the held entry is built +// with the running build module, and the value is written back with what was done. +func released(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) { + if why := b.Release(ctx, r.Module, r.FP, r.Why, r.By); why != "" { + r.Answer, r.Refused = why, true + } else { + r.Answer = "released: the entry is built with the running build module, and its wait ended" + } + r.Answered = time.Now().UTC() + data, err := json.Marshal(r) + if err != nil { + return + } + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go new file mode 100644 index 00000000..29005677 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-builder/internal/build" +) + +// waitBody is the body of wait-held and wait-ended on the builder's stream waits (design 47 G §672): the fields of §629's +// wait with `step` building, `on` toolchain:@, `module` and `fingerprint`, and, on wait-ended, `how`. +type waitBody struct { + Step string `json:"step"` + On string `json:"on"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + Since time.Time `json:"since"` + Bound string `json:"bound,omitempty"` + How string `json:"how,omitempty"` +} + +const waitStepBuilding = "building" + +// waitsKept keeps the builder's toolchain waits in its old state waits, one key . (design 47 G §672), +// and says wait-held and wait-ended on its old stream waits; at its start it says wait-held again for every key. +type waitsKept struct { + kv *process.AwaitedBucket + say func(ctx context.Context, event, id string, body any) error + bound time.Duration + logf func(string, ...any) + + mu sync.Mutex + held map[string]build.ToolchainWait +} + +func newWaitsKept(kv *process.AwaitedBucket, say func(context.Context, string, string, any) error, bound time.Duration, logf func(string, ...any)) *waitsKept { + return &waitsKept{kv: kv, say: say, bound: bound, logf: logf, held: map[string]build.ToolchainWait{}} +} + +func (w *waitsKept) body(t build.ToolchainWait, how string) waitBody { + return waitBody{Step: waitStepBuilding, On: t.On(), Module: t.Module, Fingerprint: t.Fingerprint, Since: t.Since, Bound: w.bound.String(), How: how} +} + +// start reads the state whole and says wait-held again for each key. +func (w *waitsKept) start(ctx context.Context, kv jetstream.KeyValue) { + keys, err := kv.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + w.logf("%s cannot be read, so the waits of before this start are not said again: %v", build.WaitsState, err) + return + } + for _, k := range keys { + e, err := kv.Get(ctx, k) + if err != nil { + continue + } + var t build.ToolchainWait + if json.Unmarshal(e.Value(), &t) != nil || t.Module == "" { + w.logf("%s.%s holds what is not a wait, and is left", build.WaitsState, k) + continue + } + w.mu.Lock() + w.held[k] = t + w.mu.Unlock() + w.said(ctx, "wait-held", t, "") + } +} + +func (w *waitsKept) said(ctx context.Context, event string, t build.ToolchainWait, how string) { + if err := w.say(ctx, event+"."+t.Module+"."+t.Fingerprint, event+"-"+t.Module+"-"+t.Fingerprint+"-"+t.Since.UTC().Format(time.RFC3339Nano)+how, w.body(t, how)); err != nil { + w.logf("saying %s of %s at %s: %v", event, t.Module, t.Fingerprint, err) + } +} + +// Hold keeps the wait and says wait-held, once. +func (w *waitsKept) Hold(ctx context.Context, t build.ToolchainWait) { + key, err := builds.BuilderBuildKey(t.Module, t.Fingerprint) + if err != nil { + w.logf("a wait of %s at %s has no key: %v", t.Module, t.Fingerprint, err) + return + } + w.mu.Lock() + if _, had := w.held[key]; had { + w.mu.Unlock() + return + } + w.held[key] = t + w.mu.Unlock() + if kv := w.kv.KV(); kv != nil { + data, _ := json.Marshal(t) + if _, err := kv.Put(ctx, key, data); err != nil { + w.logf("%s.%s could not be written, so the wait is kept in memory only: %v", build.WaitsState, key, err) + } + } + w.said(ctx, "wait-held", t, "") +} + +// End deletes the wait and says wait-ended with how; it answers whether one was held. +func (w *waitsKept) End(ctx context.Context, module, fp, how string) bool { + key, err := builds.BuilderBuildKey(module, fp) + if err != nil { + return false + } + w.mu.Lock() + t, had := w.held[key] + delete(w.held, key) + w.mu.Unlock() + if !had { + return false + } + if kv := w.kv.KV(); kv != nil { + if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + w.logf("%s.%s could not be deleted: %v", build.WaitsState, key, err) + } + } + w.said(ctx, "wait-ended", t, how) + return true +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index cd045595..ff63d787 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -52,7 +52,10 @@ type Builder struct { Held(ctx context.Context, a sdkstep.StepAsk) (bool, error) } Toolchains ToolchainEntries - Manifests Manifests + // Waits keeps the toolchain waits of design 47 G §672. Nil holds none: an entry whose fingerprint names another toolchain is + // left unbuilt, as before. + Waits Waits + Manifests Manifests // Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966, // novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the // builder's next start. Ended is told each key whose end was written. @@ -129,6 +132,8 @@ type Builder struct { waiting map[string]map[string]chan string // hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key. hands map[string]*handWatch + // released are the keys builder_release is building now, whatever toolchain their fingerprint names (§276, §672). + released map[string]bool // builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of // the module takes its older one's build out, and the asker drops that build's asks. builds map[string]context.CancelCauseFunc @@ -466,15 +471,22 @@ func (b *Builder) startOnce(ctx context.Context, key, module string, m meshcatal func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) { key, _ := builds.BuilderBuildKey(module, m.Fp2) defer b.handEnded(key) + defer b.unrelease(key) plan, planErr, read := b.planOf(ctx, m.Manifest) nothing := read && planErr == nil && len(plan.Tasks) == 0 - if planErr == nil && !nothing { + if planErr == nil && !nothing && !b.isReleased(key) { if why := b.stale(m, plan); why != "" { + if w, ok := b.toolchainWait(module, m, plan); ok && b.Waits != nil { + b.Waits.Hold(ctx, w) + } // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build // now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written. b.logf("%s at %s is not built: %s", module, m.Fp2, why) return } + if b.Waits != nil { + b.Waits.End(ctx, module, m.Fp2, WaitRan) // a toolchain-agreed names the version the fingerprint names + } } b.mu.Lock() trunk := b.trunkFP[key] @@ -679,7 +691,11 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module } tc := ag.Toolchain sitting = append(sitting, SittingOutWords(language, ag)...) - if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe { + released := false + if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { + released = b.isReleased(key) // builder_release builds it with the running build module (§276) + } + if want := m.Toolchains[language]; !released && (want.Version != tc.Version || want.Recipe != tc.Recipe) { return builds.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written } u := builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} diff --git a/modules/mesh-delivery-builder/internal/build/waits.go b/modules/mesh-delivery-builder/internal/build/waits.go new file mode 100644 index 00000000..bf3ca02d --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/waits.go @@ -0,0 +1,126 @@ +package build + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// **The builder's toolchain wait** (design 47 G §672, §267, §276): the builder holds an entry while its fingerprint names a +// toolchain version of its language other than the one the agreed toolchain of that language names. +const ( + // WaitsTool is builder_waits, which reads the waits; ReleaseTool is builder_release, which builds the entry with the + // running build module (§276), a hand act written in the hand-act log. + WaitsTool = "builder_waits" + ReleaseTool = "builder_release" + // WaitsState is the builder's old state section of the waits, one key .. + WaitsState = "waits" + WaitsBucket = builds.BuilderModule + "_" + WaitsState + // ModeRelease is the Mode of a reasks value of builder_release. + ModeRelease = "release" + // WaitRan ends a wait when a toolchain-agreed names the version; WaitReleased when builder_release ended it. + WaitRan = "ran" + WaitReleased = "released" +) + +// ToolchainWait is one wait: the entry's module and fingerprint, and the toolchain it waits for. +type ToolchainWait struct { + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + Language string `json:"language"` + Version string `json:"version"` + Since time.Time `json:"since"` +} + +// On is what the wait waits for, as the wait's `on` says it. +func (w ToolchainWait) On() string { return "toolchain:" + w.Language + "@" + w.Version } + +// Waits keeps the builder's waits: Hold writes one in the state waits and says wait-held, once per wait; End deletes it and +// says wait-ended with how, and answers whether one was held. +type Waits interface { + Hold(ctx context.Context, w ToolchainWait) + End(ctx context.Context, module, fp, how string) bool +} + +// toolchainWait is the toolchain wait of a module's fingerprint, or false when no language of its plan names a version +// other than the agreed one (a language with no agreed toolchain is not a wait of §672). +func (b *Builder) toolchainWait(module string, m meshcatalog.ModulesByModuleValue, plan Plan) (ToolchainWait, bool) { + for _, bm := range plan.Modules() { + if !plan.NeedsToolchain(bm) { + continue + } + language := buildrun.LanguageOf[bm] + ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language)) + if want, ok := m.Toolchains[language]; ag.Wait == "" && ok && want.Version != ag.Toolchain.Version { + return ToolchainWait{Module: module, Fingerprint: m.Fp2, Language: language, Version: want.Version, Since: b.now()}, true + } + } + return ToolchainWait{}, false +} + +// Release builds the held entry of module at fp with the running build module, now, and ends its wait released (design 47 +// G §276, §672). It answers why nothing was done, or empty. +func (b *Builder) Release(ctx context.Context, module, fp, why, by string) string { + b.init() + key, err := builds.BuilderBuildKey(module, fp) + if err != nil { + return err.Error() + } + if b.Waits == nil { + return "this builder keeps no waits" + } + b.mu.Lock() + m, known := b.known[module] + b.mu.Unlock() + if !known || m.Fp2 != fp || m.Manifest == "" { + return fmt.Sprintf("module %s does not have the fingerprint %s now: the builder builds the module's current fingerprint", module, fp) + } + b.mu.Lock() + if b.released == nil { + b.released = map[string]bool{} + } + b.released[key] = true + b.mu.Unlock() + if !b.Waits.End(ctx, module, fp, WaitReleased) { + b.mu.Lock() + delete(b.released, key) + b.mu.Unlock() + return fmt.Sprintf("%s holds no toolchain wait: %s builds a failed key again", key, BuildAgainTool) + } + b.logf("%s: released by %s (%s): built with the running build module", key, by, why) + b.startOnce(ctx, key, module, m) + return "" +} + +func (b *Builder) isReleased(key string) bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.released[key] +} + +func (b *Builder) unrelease(key string) { + b.mu.Lock() + delete(b.released, key) + b.mu.Unlock() +} + +// NewRelease is the value builder_release writes for a module's fingerprint: its key and the value. +func NewRelease(module, fp, why, by string, at time.Time) (string, []byte, error) { + key, err := builds.BuilderBuildKey(module, fp) + if err != nil { + return "", nil, err + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why the entry is released (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeRelease, Module: module, FP: fp}) + return key, data, err +} diff --git a/modules/mesh-delivery-builder/internal/build/waits_test.go b/modules/mesh-delivery-builder/internal/build/waits_test.go new file mode 100644 index 00000000..2a63722f --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/waits_test.go @@ -0,0 +1,84 @@ +package build + +import ( + "context" + "sync" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// memWaits records the waits held and ended. +type memWaits struct { + mu sync.Mutex + held map[string]ToolchainWait + ended []string +} + +func (w *memWaits) Hold(_ context.Context, t ToolchainWait) { + w.mu.Lock() + defer w.mu.Unlock() + if w.held == nil { + w.held = map[string]ToolchainWait{} + } + w.held[t.Module+"."+t.Fingerprint] = t +} + +func (w *memWaits) End(_ context.Context, module, fp, how string) bool { + w.mu.Lock() + defer w.mu.Unlock() + k := module + "." + fp + if _, ok := w.held[k]; !ok { + return false + } + delete(w.held, k) + w.ended = append(w.ended, how) + return true +} + +// Design 47 G §672, §267, §276: an entry whose fingerprint names a toolchain version other than the agreed one is held, +// built and ended `ran` when the agreed toolchain moves to that version; and one held is built with the running build +// module by Release and ended `released`. +func TestAnEntryWhoseFingerprintNamesAnotherToolchainWaitsAndIsBuiltWhenItIsAgreedOrReleased(t *testing.T) { + old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}} + t.Run("ran", func(t *testing.T) { + r := newRig(t, goBundle) + w := &memWaits{} + r.b.Waits = w + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old)) + r.b.Wait() + if len(w.held) != 1 || len(r.st.vals) != 0 { + t.Fatalf("held %v, keys %v: want one wait and nothing written", w.held, r.st.vals) + } + for _, h := range w.held { + if h.On() != "toolchain:go@go1.26.0" { + t.Errorf("the wait is on %s", h.On()) + } + } + r.tools.Set("ace", tcOf("go", "go1.26.0")) + r.tools.Set("novox", tcOf("go", "go1.26.0")) + r.b.Reconsider(context.Background()) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || len(w.held) != 0 || len(w.ended) != 1 || w.ended[0] != WaitRan { + t.Errorf("build %+v, held %v, ended %v: want done and one wait ended ran", v, w.held, w.ended) + } + }) + t.Run("released", func(t *testing.T) { + r := newRig(t, goBundle) + w := &memWaits{} + r.b.Waits = w + v := moduleValueWith(t, "modules/m", old) + r.b.Module(context.Background(), "m", v) + r.b.Wait() + if why := r.b.Release(context.Background(), "m", fp1, "the build module runs", "operator"); why != "" { + t.Fatalf("release refused: %s", why) + } + r.b.Wait() + if got := valueOf(t, r.st, r.key); got.State != builds.BuildDone || got.Toolchains["go"].Version != "go1.27.1" || len(w.ended) != 1 || w.ended[0] != WaitReleased { + t.Errorf("build %+v, ended %v: want done with the running toolchain and a wait ended released", got, w.ended) + } + if why := r.b.Release(context.Background(), "m", fp1, "again", "operator"); why == "" { + t.Error("a second release of an entry that is not held was taken") + } + }) +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 0b5e7521..b35b278e 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -50,7 +50,13 @@ type manifest struct { Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` - Logs []struct { + Streams []struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` + } `json:"streams"` + Logs []struct { Name string `json:"name"` MaxAgeDays int `json:"max-age-days"` MaxMiB int `json:"max-mib"` @@ -59,10 +65,13 @@ type manifest struct { Uses []string `json:"uses"` Invokes []string `json:"invokes"` Config map[string]struct { - Type string `json:"type"` - Description string `json:"description"` - Pattern string `json:"pattern"` - Default string `json:"default"` + Type string `json:"type"` + Description string `json:"description"` + Pattern string `json:"pattern"` + Minimum float64 `json:"minimum"` + Maximum float64 `json:"maximum"` + Trusted bool `json:"trusted"` + Default any `json:"default"` } `json:"config"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -102,7 +111,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -134,8 +143,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", - "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { - t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) + "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no // build module's toolchains; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the @@ -161,9 +170,16 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } + // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. + if len(m.Streams) != 1 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 { + t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB", m.Streams) + } + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 1 { + t.Errorf("config = %+v, want toolchain-wait-minutes of 60", m.Config) + } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) @@ -176,7 +192,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved"}) { t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved", m.Claims) } - for _, k := range []string{"streams", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + for _, k := range []string{"provides", "requires", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } @@ -202,8 +218,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" { - t.Errorf("env = %v, want the broker file alone", p["env"]) + if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES"] != "${config:toolchain-wait-minutes}" { + t.Errorf("env = %v, want the broker file and the toolchain wait's setting (design 47 G §672)", p["env"]) } for _, r := range m.Resources { if r["id"] == "settings" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 3581b792..0e428436 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -23,11 +23,13 @@ "builder_old_form", "builder_publish_again", "builder_reserved", + "builder_release", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", + "builder_waits", "builder_versions", "reserved", "builder_first_views", @@ -107,11 +109,25 @@ { "name": "bounds", "history": 1, "key-ttl": true }, { "name": "queues", "history": 1 }, { "name": "toolchains-heard", "history": 1 }, + { "name": "waits", "history": 1 }, { "name": "reserved", "history": 1 }, { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, { "name": "first-views", "history": 1 } ], + "streams": [ + { "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 } + ], + "config": { + "toolchain-wait-minutes": { + "type": "number", + "description": "minutes an entry waits for the agreed toolchain of its language to name the version its fingerprint names (design 47 G section 672); a bound a whole test sets in seconds.", + "minimum": 1, + "maximum": 10080, + "trusted": false, + "default": 60 + } + }, "logs": [ { "name": "build-log", "max-age-days": 30, "max-mib": 3072 } ], @@ -156,7 +172,8 @@ "./mesh-delivery-builder" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES": "${config:toolchain-wait-minutes}" }, "user": "mesh-delivery-builder", "health": {