From a2dfbec30da3dac902dd9237dd4f228b2c761af8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 00:10:29 +0100 Subject: [PATCH] The builder no longer watches the gate's source for candidate base merges A candidate comes by the runner's build step ask (A-97), so the builder's trunkSource watch of the gate's source, its sourceHeads, the wake it gave a floor, its read mesh-delivery-gate.source and the bed's source bucket go. Builds: design 47 G A-97, A-84 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../catalogholder_integration_test.go | 17 +++-- .../mesh-delivery-builder/integration_test.go | 20 +----- .../cmd/mesh-delivery-builder/main.go | 63 ----------------- .../cmd/mesh-delivery-builder/names.go | 6 -- .../cmd/mesh-delivery-builder/names_test.go | 3 +- .../cmd/mesh-delivery-builder/optional.go | 68 ------------------- .../cmd/mesh-delivery-builder/refused_test.go | 12 ---- .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 3 +- 9 files changed, 17 insertions(+), 177 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go index 24600e96..699e8cb4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go @@ -86,12 +86,19 @@ func (h *catalogHolder) whole() meshcatalog.RevisionAnswer { } // put sets one module's entry in a new trunk revision of main and says catalog-revision. -func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { +func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { h.putAll(t, e) } + +// putAll sets several modules' entries in one new trunk revision of main. +func (h *catalogHolder) putAll(t *testing.T, es ...meshcatalog.CatalogEntry) { t.Helper() h.mu.Lock() h.revision++ - h.entries[e.Module] = e - ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: []string{e.Module}, V: 1}) + var changed []string + for _, e := range es { + h.entries[e.Module] = e + changed = append(changed, e.Module) + } + ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: changed, V: 1}) h.mu.Unlock() if _, err := h.js.Publish(h.ctx, meshcatalog.SubjectCatalogRevision("main"), ev); err != nil { t.Fatal(err) @@ -137,8 +144,10 @@ func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[stri t.Fatal(err) } t.Cleanup(func() { _ = s.Unsubscribe() }) + var entries []meshcatalog.CatalogEntry for seat, module := range holders { - h.put(t, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:claims-" + module, + entries = append(entries, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:" + fmt.Sprintf("%064x", len(module)*7919+int(module[len(module)-1])), Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}}) } + h.putAll(t, entries...) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index d53136cf..cdbc6a29 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -28,15 +28,12 @@ import ( runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" - "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views/viewstest" @@ -201,23 +198,9 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { } } -// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit. +// trunkHead is a trunk head of novox/mesh-catalog. const trunkHead = "3333333333333333333333333333333333333333" -// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision. -func (b *bed) trunk(t *testing.T, head string) uint64 { - t.Helper() - data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{V: keys.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}}) - if err != nil { - t.Fatal(err) - } - rev, err := b.source.Put(b.ctx, meshdeliveryclient.SourceKey("main"), data) - if err != nil { - t.Fatal(err) - } - return rev -} - type bed struct { ctx context.Context js jetstream.JetStream @@ -298,7 +281,6 @@ func startBedWith(t *testing.T, closed bool) *bed { } return kv } - b.source = mk(sourceBucket, sourceHistory) // The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's // verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2 // each have their durable consumer. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 29797124..3edd0cb5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -10,13 +10,10 @@ import ( "fmt" "io" "log" - "maps" "os" "os/signal" - "slices" "strconv" "strings" - "sync" "syscall" "time" @@ -31,12 +28,10 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" - "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" @@ -236,18 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return fmt.Sprintf("built again as revision %d", r.Revision), false }} - // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base - // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision - // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it - // cannot be read the trunk's keys are built and the candidates wait (issue 1945). errs := make(chan error, 10) - trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) { - // A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*). - (&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "") - b.TrunkMoved(ctx) - }, errs: errs, logf: logger.Printf} - trunks.start() - b.Trunk = trunks.head b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf} b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the @@ -505,50 +489,3 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder's own state of its first views (views.StatusState); its findings (findingsBucket) are where the condition // keeper raises a first view unanswered past its bound (ADR 0615 §6). var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState - -// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest -// sourceHistory of them: what a candidate's base revision names. -type sourceHeads struct { - mu sync.Mutex - byRev map[uint64]map[string]string -} - -// put takes one revision in and answers whether it is new. -func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool { - if !opOf(e) { - return false - } - src, err := meshdeliveryclient.DecodeSource(e.Value()) - if err != nil { - return false - } - s.mu.Lock() - defer s.mu.Unlock() - if s.byRev == nil { - s.byRev = map[uint64]map[string]string{} - } - if _, had := s.byRev[e.Revision()]; had { - return false - } - s.byRev[e.Revision()] = src.Heads - for len(s.byRev) > sourceHistory { - delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev)))) - } - return true -} - -// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more). -func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) { - s.mu.Lock() - defer s.mu.Unlock() - heads, ok := s.byRev[rev] - if !ok { - return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev) - } - for r, head := range heads { - if trunk.SameRepository(r, repository) { - return head, nil - } - } - return "", nil -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go index 53c8b837..1c4b475a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go @@ -18,12 +18,6 @@ const ( indexerModule = "mesh-delivery-indexer" // assignerModule is the holder of the state assignments. assignerModule = buildrun.AssignerModule - - // sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names. - sourceBucket = gateModule + "_source" - - // The revisions the gate's source keeps per key (the bed of the integration test creates it alike). - sourceHistory = 64 ) // namedTreePrefix opens the tree input of a module whose every artifact names its paths. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go index 00b38e80..0e646bd0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go @@ -26,7 +26,6 @@ func TestTheNamesAreTheManifests(t *testing.T) { t.Fatal(err) } for _, read := range []string{ - gateModule + ".source", "node-build-go.steps", "node-build-image.steps", "node-build-node.steps", } { if !slices.Contains(m.Reads, read) { @@ -45,7 +44,7 @@ func TestTheNamesAreTheManifests(t *testing.T) { } } for want, got := range map[string]string{ - "mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), + "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket, } { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go index 87ba0c67..7bfb1320 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go @@ -1,76 +1,8 @@ package main -import ( - "context" - "fmt" - "sync" - - "github.com/nats-io/nats.go/jetstream" -) - // **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of // mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant // reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet: // exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds, // the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left, // and opened again at the next event that needs it where there is one. - -// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and, -// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys, -// and a candidate's preview waits, saying why. -type trunkSource struct { - ctx context.Context - js bucketOpener - heads *sourceHeads - moved func(context.Context) - errs chan<- error - logf func(string, ...any) - - mu sync.Mutex - opened bool -} - -// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask. -func (t *trunkSource) ensure() error { - t.mu.Lock() - defer t.mu.Unlock() - if t.opened { - return nil - } - kv, err := t.js.KeyValue(t.ctx, sourceBucket) - if err != nil { - return fmt.Errorf("the bucket %s cannot be read (%v)", sourceBucket, err) - } - w, err := kv.WatchAll(t.ctx, jetstream.IncludeHistory()) // every watched branch's key: a revision names its head whichever branch it is of (design 47 G §15) - if err != nil { - return fmt.Errorf("the bucket %s cannot be watched (%v)", sourceBucket, err) - } - sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) }) - if err != nil { - return fmt.Errorf("the bucket %s: %w", sourceBucket, err) - } - t.opened = true - go func() { - t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) { - if t.heads.put(e) && t.moved != nil { - t.moved(t.ctx) - } - }) - }() - return nil -} - -// start opens the source at start; one that cannot be is said, and the builder goes on. -func (t *trunkSource) start() { - if err := t.ensure(); err != nil { - t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err) - } -} - -// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet. -func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) { - if err := t.ensure(); err != nil { - return "", err - } - return t.heads.head(ctx, rev, repository) -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index dee20974..34d6449c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -43,23 +43,11 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard) t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding) t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard) - var logs bytes.Buffer - logger := log.New(&logs, "", 0) refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")} if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) } - // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk - // head is told why and opens it again then. - src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} - src.start() - if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), sourceBucket) { - t.Errorf("a trunk head with the source refused: %v, want why", err) - } - if src.opened { - t.Error("a refused source counts as opened, so it is never opened again") - } t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads) } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 807e7a61..950de63c 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -118,7 +118,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a // node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3). - if want := []string{"mesh-delivery-assigner.assignments", "mesh-delivery-gate.source", + if want := []string{"mesh-delivery-assigner.assignments", "node-build-go.steps", "node-build-image.steps", "node-build-node.steps"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 49972d01..0336fd6f 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -142,8 +142,7 @@ "node-build-go.steps", "node-build-image.steps", "node-build-node.steps", - "mesh-delivery-assigner.assignments", - "mesh-delivery-gate.source" + "mesh-delivery-assigner.assignments" ], "uses": [ "node-build-go",