novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

The builder no longer watches the gate's source for candidate base merges

Browse Source 
A candidate comes by the runner's build step ask (A-97), so the builder's trunkSource watch of the gate's source, its
sourceHeads, the wake it gave a floor, its read mesh-delivery-gate.source and the bed's source bucket go.

Builds: design 47 G A-97, A-84
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
jschoubben
2026-11-12 00:10:29 +01:00
parent 8160dc8d26
commit a2dfbec30d
9 changed files with 17 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go
+13 -4
View File
@@ -86,12 +86,19 @@ func (h *catalogHolder) whole() meshcatalog.RevisionAnswer {
}
// put sets one module's entry in a new trunk revision of main and says catalog-revision.
func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) {
func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { h.putAll(t, e) }
// putAll sets several modules' entries in one new trunk revision of main.
func (h *catalogHolder) putAll(t *testing.T, es ...meshcatalog.CatalogEntry) {
t.Helper()
h.mu.Lock()
h.revision++
h.entries[e.Module] = e
ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: []string{e.Module}, V: 1})
var changed []string
for _, e := range es {
h.entries[e.Module] = e
changed = append(changed, e.Module)
}
ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: changed, V: 1})
h.mu.Unlock()
if _, err := h.js.Publish(h.ctx, meshcatalog.SubjectCatalogRevision("main"), ev); err != nil {
t.Fatal(err)
@@ -137,8 +144,10 @@ func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[stri
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Unsubscribe() })
var entries []meshcatalog.CatalogEntry
for seat, module := range holders {
h.put(t, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:claims-" + module,
entries = append(entries, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:" + fmt.Sprintf("%064x", len(module)*7919+int(module[len(module)-1])),
Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}})
}
h.putAll(t, entries...)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+1 -19
View File
@@ -28,15 +28,12 @@ import (
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
@@ -201,23 +198,9 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
}
}
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
// trunkHead is a trunk head of novox/mesh-catalog.
const trunkHead = "3333333333333333333333333333333333333333"
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
func (b *bed) trunk(t *testing.T, head string) uint64 {
t.Helper()
data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{V: keys.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
if err != nil {
t.Fatal(err)
}
rev, err := b.source.Put(b.ctx, meshdeliveryclient.SourceKey("main"), data)
if err != nil {
t.Fatal(err)
}
return rev
}
type bed struct {
ctx context.Context
js jetstream.JetStream
@@ -298,7 +281,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
}
return kv
}
b.source = mk(sourceBucket, sourceHistory)
// The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's
// verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2
// each have their durable consumer.
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
-63
View File
@@ -10,13 +10,10 @@ import (
"fmt"
"io"
"log"
"maps"
"os"
"os/signal"
"slices"
"strconv"
"strings"
"sync"
"syscall"
"time"
@@ -31,12 +28,10 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -236,18 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
return fmt.Sprintf("built again as revision %d", r.Revision), false
}}
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
errs := make(chan error, 10)
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) {
// A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*).
(&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "")
b.TrunkMoved(ctx)
}, errs: errs, logf: logger.Printf}
trunks.start()
b.Trunk = trunks.head
b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf}
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465)
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
@@ -505,50 +489,3 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The builder's own state of its first views (views.StatusState); its findings (findingsBucket) are where the condition
// keeper raises a first view unanswered past its bound (ADR 0615 §6).
var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
// sourceHistory of them: what a candidate's base revision names.
type sourceHeads struct {
mu sync.Mutex
byRev map[uint64]map[string]string
}
// put takes one revision in and answers whether it is new.
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
if !opOf(e) {
return false
}
src, err := meshdeliveryclient.DecodeSource(e.Value())
if err != nil {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if s.byRev == nil {
s.byRev = map[uint64]map[string]string{}
}
if _, had := s.byRev[e.Revision()]; had {
return false
}
s.byRev[e.Revision()] = src.Heads
for len(s.byRev) > sourceHistory {
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
}
return true
}
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
heads, ok := s.byRev[rev]
if !ok {
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
}
for r, head := range heads {
if trunk.SameRepository(r, repository) {
return head, nil
}
}
return "", nil
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go
-6
View File
@@ -18,12 +18,6 @@ const (
indexerModule = "mesh-delivery-indexer"
// assignerModule is the holder of the state assignments.
assignerModule = buildrun.AssignerModule
// sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names.
sourceBucket = gateModule + "_source"
// The revisions the gate's source keeps per key (the bed of the integration test creates it alike).
sourceHistory = 64
)
// namedTreePrefix opens the tree input of a module whose every artifact names its paths.
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go
+1 -2
View File
@@ -26,7 +26,6 @@ func TestTheNamesAreTheManifests(t *testing.T) {
t.Fatal(err)
}
for _, read := range []string{
gateModule + ".source",
"node-build-go.steps", "node-build-image.steps", "node-build-node.steps",
} {
if !slices.Contains(m.Reads, read) {
@@ -45,7 +44,7 @@ func TestTheNamesAreTheManifests(t *testing.T) {
}
}
for want, got := range map[string]string{
"mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"),
"mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"),
buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard",
"mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket,
} {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go
-68
View File
@@ -1,76 +1,8 @@
package main
import (
"context"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
)
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
// and opened again at the next event that needs it where there is one.
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
// and a candidate's preview waits, saying why.
type trunkSource struct {
ctx context.Context
js bucketOpener
heads *sourceHeads
moved func(context.Context)
errs chan<- error
logf func(string, ...any)
mu sync.Mutex
opened bool
}
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
func (t *trunkSource) ensure() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.opened {
return nil
}
kv, err := t.js.KeyValue(t.ctx, sourceBucket)
if err != nil {
return fmt.Errorf("the bucket %s cannot be read (%v)", sourceBucket, err)
}
w, err := kv.WatchAll(t.ctx, jetstream.IncludeHistory()) // every watched branch's key: a revision names its head whichever branch it is of (design 47 G §15)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched (%v)", sourceBucket, err)
}
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
if err != nil {
return fmt.Errorf("the bucket %s: %w", sourceBucket, err)
}
t.opened = true
go func() {
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
if t.heads.put(e) && t.moved != nil {
t.moved(t.ctx)
}
})
}()
return nil
}
// start opens the source at start; one that cannot be is said, and the builder goes on.
func (t *trunkSource) start() {
if err := t.ensure(); err != nil {
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
}
}
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
if err := t.ensure(); err != nil {
return "", err
}
return t.heads.head(ctx, rev, repository)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go
-12
View File
@@ -43,23 +43,11 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard)
t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding)
t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard)
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), sourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads)
}
modules/mesh-delivery-builder/manifest_test.go
+1 -1
View File
@@ -118,7 +118,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a
// node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3).
if want := []string{"mesh-delivery-assigner.assignments", "mesh-delivery-gate.source",
if want := []string{"mesh-delivery-assigner.assignments",
"node-build-go.steps", "node-build-image.steps", "node-build-node.steps"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
modules/mesh-delivery-builder/module.json
+1 -2
View File
@@ -142,8 +142,7 @@
"node-build-go.steps",
"node-build-image.steps",
"node-build-node.steps",
"mesh-delivery-assigner.assignments",
"mesh-delivery-gate.source"
"mesh-delivery-assigner.assignments"
],
"uses": [
"node-build-go",