From cfd2afe39c39c4b7b4456a5d80b3a4080164fa18 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 20:25:03 +0100 Subject: [PATCH] H-A-492: the record's bound is a setting; a build key read error is said Review round 1 (principle 11, configurable, never hard-coded): the bound of the record of a version's digests is the builder's setting record-bound-seconds (default 60), read from its file as revision-wait and package-wait are, set on the Builder and used for the record. A build key the builder cannot read, or that holds no build, is said in its journal instead of being taken silently as not done (principle 3, failure is loud). Builds: design 47 H2 H-A-492 --- .../cmd/mesh-delivery-builder/main.go | 6 +++ .../internal/build/numbers.go | 30 ++++++++++----- .../internal/build/reserve_test.go | 38 +++++++++++++++++++ .../internal/build/run.go | 5 ++- .../mesh-delivery-builder/manifest_test.go | 24 +++++++----- modules/mesh-delivery-builder/module.json | 17 +++++++++ 6 files changed, 100 insertions(+), 20 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 290698c49..a2b0d1000 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -233,6 +233,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } b.PackageWait = time.Duration(packageWait) * time.Minute + // A version's digests are recorded under the builder's own bound, never the build's (H-A-492); it is record-bound-seconds. + recordBound, err := numberFile(getenv("MESH_BUILDER_RECORD_BOUND_FILE"), int(build.DefaultRecordBound/time.Second)) + if err != nil { + return err + } + b.RecordBound = time.Duration(recordBound) * time.Second // Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194). b.PublishedVersions = publishedVersionsVia(calls.Call) waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 421c3c954..d7ffa9a8e 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -106,11 +106,19 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string return nil } raw, _, found, err := b.Claimer.Store.Get(ctx, key) - if err != nil || !found { + if err != nil { + b.logf("%s: the build key cannot be read, so it is taken as not done: %v", key, err) + return nil + } + if !found { return nil } cur, err := builds.DecodeBuilderBuild(raw) - if err != nil || cur.State != builds.BuildDone { + if err != nil { + b.logf("%s: the build key holds what is no build, so it is taken as not done: %v", key, err) + return nil + } + if cur.State != builds.BuildDone { return nil } out := map[string]string{} @@ -122,14 +130,18 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string return out } -// recordBound bounds the record of a reservation's digests (design 47 H2 H-A-492): the builder's own bound, never the build's -// context. -const recordBound = time.Minute +// DefaultRecordBound is the bound of the record of a reservation's digests, its setting record-bound-seconds' default (design 47 +// H2 H-A-492). +const DefaultRecordBound = time.Minute // recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by -// recordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record. -func recordContext(ctx context.Context) (context.Context, context.CancelFunc) { - return context.WithTimeout(context.WithoutCancel(ctx), recordBound) +// RecordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record. +func (b *Builder) recordContext(ctx context.Context) (context.Context, context.CancelFunc) { + bound := b.RecordBound + if bound <= 0 { + bound = DefaultRecordBound + } + return context.WithTimeout(context.WithoutCancel(ctx), bound) } // RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47 @@ -160,7 +172,7 @@ func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (str } refs[ref] = d } - rec, stop := recordContext(ctx) + rec, stop := b.recordContext(ctx) defer stop() if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil { return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err) diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index 839554731..87341d179 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -186,6 +186,37 @@ func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) { t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built) } }) + t.Run("the record's bound is the setting record-bound-seconds, never the build's", func(t *testing.T) { + build, cancel := context.WithCancel(context.Background()) + cancel() + for _, c := range []struct { + bound, want time.Duration + }{{0, DefaultRecordBound}, {5 * time.Second, 5 * time.Second}} { + rec, stop := (&Builder{RecordBound: c.bound}).recordContext(build) + deadline, has := rec.Deadline() + stop() + if left := time.Until(deadline); !has || left > c.want || left < c.want-time.Second { + t.Errorf("RecordBound %v: the record's context has deadline %v (%v left, %v), want %v left of its own", c.bound, deadline, left, has, c.want) + } + } + rec, stop := (&Builder{}).recordContext(build) + defer stop() + if rec.Err() != nil { + t.Errorf("the record's context ended with the build's: %v", rec.Err()) + } + }) + t.Run("a build key that cannot be read is said, not taken silently", func(t *testing.T) { + var journal []string + b := &Builder{Claimer: Claimer{Store: unreadable{newMem()}}, Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }} + if d := b.doneDigests(context.Background(), "m", fp1); d != nil { + t.Errorf("an unreadable key answered digests %v", d) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "cannot be read") && strings.Contains(l, "bus timeout") + }) { + t.Errorf("journal %q, want a line saying the build key cannot be read, with why", journal) + } + }) t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) { r := newRig(t, goBundle) r.mf.byDigest[mdigest('a')] = manifest @@ -217,6 +248,13 @@ func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) { }) } +// unreadable is a builds bucket whose reads fail, as one on a bus that does not answer. +type unreadable struct{ *memStore } + +func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) { + return nil, 0, false, errors.New("bus timeout") +} + // TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through // reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why. func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e7acff14d..5242f95ce 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -127,6 +127,9 @@ type Builder struct { // package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished) // gives a pin (H-A-192). PackageWait time.Duration + // RecordBound bounds the record of a reservation's digests after a build ends, its setting record-bound-seconds (design 47 + // H2 H-A-492): the builder's own bound, never the build's context; DefaultRecordBound when 0. + RecordBound time.Duration // PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first // (H-A-194): read once before a build waits for that package. Nil reads none. PublishedVersions func(ctx context.Context, path string) ([]string, error) @@ -867,7 +870,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed // build unrecorded, so unreleased (design 47 H2 H-A-492). - rec, stop := recordContext(ctx) + rec, stop := b.recordContext(ctx) err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key)) stop() if err != nil { diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 0f3fbe919..b09cdf013 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -199,15 +199,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 || + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 13 || // design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" || !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || // H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || + // H-A-492: the record of a version's digests runs under the builder's own bound, a setting of 60 s by default + m.Config["record-bound-seconds"].Default != float64(60) || m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 { - t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config) + t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), record-bound-seconds of 60 (H-A-492), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config) } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { @@ -266,7 +268,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("health = %v, want kind unit alone", p["health"]) } env, _ := p["env"].(map[string]any) - if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { + if len(env) != 9 || env["MESH_BUILDER_RECORD_BOUND_FILE"] != "${dir:state}/record-bound-seconds" || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) } for k, v := range env { @@ -274,14 +276,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v) } } - found := false - for _, r := range m.Resources { - if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" { - found = true + for _, setting := range []string{"toolchain-wait-minutes", "record-bound-seconds"} { // H-A-492: the record's bound too + found := false + for _, r := range m.Resources { + if r["type"] == "file" && r["path"] == "${dir:state}/"+setting && r["content"] == "${config:"+setting+"}" { + found = true + } + } + if !found { + t.Errorf("no file resource with the content ${config:%s}: A-71", setting) } - } - if !found { - t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71") } for _, r := range m.Resources { if r["id"] == "settings" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f922c7b79..7c9f4e6e9 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -183,6 +183,14 @@ "trusted": false, "default": 10 }, + "record-bound-seconds": { + "type": "number", + "description": "seconds the builder gives the record of a version's digests after its build ends, under its own bound and never the build's (design 47 H2 H-A-492).", + "minimum": 1, + "maximum": 3600, + "trusted": false, + "default": 60 + }, "ask-taken-bound": { "type": "string", "description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).", @@ -310,6 +318,14 @@ "mode": "0600", "content": "${config:package-wait-minutes}" }, + { + "id": "record-bound", + "type": "file", + "path": "${dir:state}/record-bound-seconds", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:record-bound-seconds}" + }, { "id": "publish-pause", "type": "file", @@ -361,6 +377,7 @@ "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes", "MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes", "MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes", + "MESH_BUILDER_RECORD_BOUND_FILE": "${dir:state}/record-bound-seconds", "MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause", "MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max", "MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",