From efcc8125af67e93c269aa1ea9da0eb6b44d986b9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:09:33 +0100 Subject: [PATCH 1/9] BL9: a group run's build names each package member's build by its digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A candidate's module is fetched against each package member of its group by the digest of the member's build (fetch `builds`), the member out of the fetched packages; a member not built holds it with wait-held naming the member, its bound a key of the builder's bounds state with its own time to live (A-77), a failed member or one missing past the bound fails the step naming it. Builds: design 47 G §373, §465 (A-77) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/main.go | 8 + .../cmd/mesh-delivery-builder/memberbounds.go | 42 ++++ .../internal/build/candidates.go | 203 +++++++++++++++++- .../internal/build/members_test.go | 166 ++++++++++++++ .../internal/build/run.go | 24 +++ .../internal/versions/names.go | 32 +++ .../internal/versions/publish.go | 11 +- 7 files changed, 483 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go create mode 100644 modules/mesh-delivery-builder/internal/build/members_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index b873b4c2..adac30c3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }, errs: errs, logf: logger.Printf} trunks.start() b.Trunk = trunks.head + b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf} + b.Members = versions.ModuleOf(indexerPackages{packages}) // the build of a group's package member (design 47 G §373, §465) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the // indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since. b.Log = func(e builds.BuildLogEntry) { @@ -419,6 +421,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput.expired(ctx, e) return } + if key, ok := versions.MemberBoundOf(e.Key()); ok { + if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member + b.MemberBoundEnded(ctx, key) + } + return + } if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended reads.expiredCall(node) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go new file mode 100644 index 00000000..63e15b10 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/versions" +) + +// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of +// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded). +// No timer. +type memberBounds struct { + bounds func() jetstream.KeyValue + now func() time.Time + logf func(string, ...any) +} + +// Arm is build.MemberBounds' Arm. +func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error { + kv := m.bounds() + if kv == nil { + return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) + } + if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { + return err + } + return nil +} + +// End is build.MemberBounds' End. +func (m *memberBounds) End(ctx context.Context, key string) { + if kv := m.bounds(); kv != nil { + if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + m.logf("the member bound of %s is not deleted: %v", key, err) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index fc65815c..2cf7d3e4 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -3,7 +3,11 @@ package build import ( "context" "fmt" + "maps" + "slices" "sort" + "strings" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -20,7 +24,15 @@ import ( // via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk // commit the head is merged onto. -type via struct{ Ref, Base string } +type via struct { + Ref, Base string + // Members are the group's package members the module is built against by their builds (design 47 G §373, §465). + Members []member +} + +// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's +// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate. +type member struct{ Package, Module, Artifact, Fp string } // Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it // carries. A value that is not a candidate is said and left. @@ -46,6 +58,14 @@ func (b *Builder) ForgetCandidate(key string) { } func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) { + group := map[string]string{} // the fingerprint of each module the group's pull requests change + for _, p := range c.Pulls { + for name, cm := range p.Fps { + if cm.Fp2 != "" && cm.Manifest != "" { + group[name] = cm.Fp2 + } + } + } for _, p := range c.Pulls { names := make([]string, 0, len(p.Fps)) for name := range p.Fps { @@ -83,9 +103,14 @@ func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.Candidate b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) continue } + members, err := b.membersOf(ctx, name, cm, group) + if err != nil { + b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err) + continue + } b.mu.Lock() if _, onTrunk := b.trunkFP[key]; !onTrunk { - b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base} + b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members} } b.mu.Unlock() b.consider(ctx, name, m) @@ -131,3 +156,177 @@ func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) } return head, nil } + +// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package +// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the +// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds, +// is no member: it is fetched as before. +func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) { + if b.Members == nil { + return nil, nil + } + var out []member + for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) { + if len(cm.Packages[pkg]) != 40 { + continue + } + module, artifact, found, err := b.Members(ctx, pkg) + if err != nil { + return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err) + } + if !found || module == name || group[module] == "" { + continue + } + out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]}) + } + return out, nil +} + +// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's +// 60-minute bound). +const MemberBound = 60 * time.Minute + +// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77): +// Arm writes it for key, the dependent's build key; End deletes it. +type MemberBounds interface { + Arm(ctx context.Context, key, why string, bound time.Duration) error + End(ctx context.Context, key string) +} + +// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the +// candidates are looked at again, which fails its step naming the member still not built. +func (b *Builder) MemberBoundEnded(ctx context.Context, key string) { + b.init() + b.mu.Lock() + waiting := b.memberWaits[key] + if waiting { + b.memberExpired[key] = true + } + b.mu.Unlock() + if waiting { + b.TrunkMoved(ctx) + } +} + +// endMemberWait forgets key's wait and deletes its bound. +func (b *Builder) endMemberWait(ctx context.Context, key string) { + b.mu.Lock() + had := b.memberWaits[key] + delete(b.memberWaits, key) + delete(b.memberExpired, key) + b.mu.Unlock() + if had && b.MemberBounds != nil { + b.MemberBounds.End(ctx, key) + } +} + +// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the +// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or +// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the +// builder holds no timer. +func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) { + b.mu.Lock() + v := b.via[key] + b.mu.Unlock() + if len(v.Members) == 0 { + return nil, "", false + } + defer func() { + if !waiting { + b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound + } + }() + var pending []string + for _, mem := range v.Members { + ref := mem.Module + "/" + mem.Artifact + mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp) + if err != nil { + return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false + } + raw, _, found, err := b.Claimer.Store.Get(ctx, mk) + if err != nil { + return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false + } + if !found { + pending = append(pending, ref) + continue + } + cur, err := builds.DecodeBuilderBuild(raw) + if err != nil { + return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false + } + switch cur.State { + case builds.BuildDone: + digest := "" + for _, a := range cur.Artifacts { + if a.Name == mem.Artifact { + digest = a.Digest + } + } + if digest == "" { + return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false + } + got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest}) + case builds.BuildFailed: + return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false + default: + pending = append(pending, ref) + } + } + if len(pending) == 0 { + b.mu.Lock() + b.fetched[key] = got + b.mu.Unlock() + return got, "", false + } + b.mu.Lock() + said := b.memberWaits[key] + expired := b.memberExpired[key] + b.mu.Unlock() + if expired { + return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false + } + if !said { + line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String() + if b.MemberBounds != nil { + // The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written + // would never end, so the step fails at once, saying so. + if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil { + return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false + } + } + b.mu.Lock() + b.memberWaits[key] = true + b.mu.Unlock() + b.logf("%s", line) + if b.Log != nil { + b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"}) + } + } + return nil, "", true +} + +// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's +// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits. +func (b *Builder) memberEnded(ctx context.Context, key string) { + b.mu.Lock() + member := false + for _, v := range b.via { + for _, mem := range v.Members { + if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key { + member = true + } + } + } + b.mu.Unlock() + if member { + b.TrunkMoved(ctx) + } +} + +// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none. +func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild { + b.mu.Lock() + defer b.mu.Unlock() + return b.fetched[key] +} diff --git a/modules/mesh-delivery-builder/internal/build/members_test.go b/modules/mesh-delivery-builder/internal/build/members_test.go new file mode 100644 index 00000000..7354e477 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/members_test.go @@ -0,0 +1,166 @@ +package build + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const memberPackage = "example.org/p" + +var fpP = "src2:" + hex64('2') + +// groupValue is a group's candidate: the module m, built against the head of the package member p, and p. +func groupValue(t *testing.T) []byte { + t.Helper() + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{ + "m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}}, + "p": {Fp2: fpP, Manifest: mdigest('b')}}}} + hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}}) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + return data +} + +// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known. +func memberRig(t *testing.T, known bool) *rig { + t.Helper() + r := newRig(t, goBundle) + lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"} + r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}} + r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) { + if pkg == memberPackage { + return "p", "lib", true, nil + } + return "", "", false, nil + } + trunkModule(t, r) + if known { + data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'), + Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)}) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "p", data) + r.b.Wait() + } + return r +} + +func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103 + t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) { + r := memberRig(t, true) + r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { + t.Fatalf("m = %+v, want built", v) + } + var found bool + for _, c := range r.bm.callsOf("build_go_fetch") { + f := c.body.(map[string]any) + bs, _ := f["builds"].([]any) + if len(bs) == 0 { + continue + } + found = true + want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest} + if len(bs) != 1 || !jsonEqual(bs[0], want) { + t.Errorf("builds = %v, want %v", bs, want) + } + pk, _ := f["packages"].(map[string]any) + if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" { + t.Errorf("packages = %v, want the member out of it and the other package kept", pk) + } + } + if !found { + t.Error("no fetch named the member's build") + } + }) + t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) { + r := memberRig(t, false) + var mu sync.Mutex + bounds := &fakeMemberBounds{} + r.b.MemberBounds = bounds + var said []builds.BuildLogEntry + r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() } + r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) + r.b.Wait() + if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 { + t.Fatal("the dependent was claimed or built while its package member is not built") + } + mu.Lock() + if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") { + t.Errorf("said = %+v, want one wait-held naming p/lib", said) + } + mu.Unlock() + if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() { + t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got) + } + r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry + r.b.Wait() + if got := bounds.ended(); len(got) != 1 || got[0] != r.key { + t.Errorf("bounds ended = %v, want the wait's bound deleted", got) + } + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 { + t.Errorf("m = %+v, want failed naming the member and nothing built", v) + } + }) + t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) { + r := memberRig(t, false) + mk, _ := builds.BuilderBuildKey("p", fpP) + data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"}) + if err != nil { + t.Fatal(err) + } + r.st.put(mk, data) + r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t)) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 { + t.Errorf("m = %+v, want failed naming the member and its error", v) + } + }) +} + +// fakeMemberBounds keeps the bounds a builder arms and ends. +type fakeMemberBounds struct { + mu sync.Mutex + arms, ends_ []string +} + +func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error { + f.mu.Lock() + defer f.mu.Unlock() + f.arms = append(f.arms, key+" "+bound.String()) + return nil +} + +func (f *fakeMemberBounds) End(_ context.Context, key string) { + f.mu.Lock() + defer f.mu.Unlock() + f.ends_ = append(f.ends_, key) +} + +func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms } +func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ } + +func jsonEqual(a, b any) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ff63d787..52402be3 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -74,6 +74,11 @@ type Builder struct { // Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a // candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why. Trunk func(ctx context.Context, rev uint64, repository string) (string, error) + // Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G + // §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a + // member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none. + Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error) + MemberBounds MemberBounds // Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the // ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked // again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it. @@ -125,6 +130,10 @@ type Builder struct { // published are the package versions `@` a done key of this bucket names as a package's reference: a key // waiting for one is built once it is here (ADR 0500 clause 6). published map[string]bool + // memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired + // those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go). + memberWaits, memberExpired map[string]bool + fetched map[string][]sdkstep.FetchBuild // held are the done keys' references the store answered held, by ` `, asked once per process: a // blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937). held map[string]bool @@ -163,6 +172,7 @@ func (b *Builder) init() { b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands = map[string]*handWatch{} + b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{} b.builds = map[string]context.CancelCauseFunc{} }) } @@ -501,6 +511,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } } + memberFail := "" + if !trunk && planErr == nil && !nothing { + var waiting bool + if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting { + return // a package member of the group is not built (design 47 G §373): its end looks at this key again + } + } reserving := trunk && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) if reserving && b.waitFill(module, m) { return @@ -549,6 +566,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB // Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key // fails with why, and is taken again at the next look. result = failed("%s", numbering.Refused) + } else if memberFail != "" && cl.Build.State != builds.BuildDone { + result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373) } else if nothing { // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). @@ -595,6 +614,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB if b.Ended != nil { b.Ended(ctx, cl.Key) } + b.memberEnded(ctx, cl.Key) ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result) ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26) b.logBuild(ended) @@ -932,6 +952,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes b.mu.Unlock() if candidate { req.Ref, req.Base = v.Ref, v.Base + req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465) } } if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 { @@ -968,6 +989,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes // The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes // them into the workspace, whatever the repository's go.mod names. ask.Packages = maps.Clone(m.Packages) + for _, fb := range req.Builds { + delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) + } } if id := asked.resume(language); id != "" { ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it diff --git a/modules/mesh-delivery-builder/internal/versions/names.go b/modules/mesh-delivery-builder/internal/versions/names.go index 66dd54d0..e6e7d091 100644 --- a/modules/mesh-delivery-builder/internal/versions/names.go +++ b/modules/mesh-delivery-builder/internal/versions/names.go @@ -70,3 +70,35 @@ func isSeriesKey(key string) bool { _, _, ok := meshcatalog.ParsePackagesByPackageSeriesKey(key) return ok } + +// ModuleOf is the module and artifact a package's name in its ecosystem is built from, as the indexer's packages state holds +// them (the inverse of NamesIn; design 47 G §373, §465): the series key whose value names that name. +func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { + return func(ctx context.Context, name string) (string, string, bool, error) { + keys, err := s.Keys(ctx) + if err != nil { + return "", "", false, err + } + sort.Strings(keys) + for _, key := range keys { + if !isSeriesKey(key) { + continue + } + value, found, err := s.Get(ctx, key) + if err != nil { + return "", "", false, err + } + if !found { + continue + } + v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value) + if err != nil || v.Name != name { + continue + } + if module, artifact, cut := strings.Cut(v.Package, "/"); cut && module != "" && artifact != "" { + return module, artifact, true, nil + } + } + return "", "", false, nil + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/publish.go b/modules/mesh-delivery-builder/internal/versions/publish.go index 7b401b8a..7384de9c 100644 --- a/modules/mesh-delivery-builder/internal/versions/publish.go +++ b/modules/mesh-delivery-builder/internal/versions/publish.go @@ -37,8 +37,16 @@ const ( publishBoundPrefix = "publish." oldFormBoundPrefix = "old-form." unputBoundPrefix = "unput." + memberBoundPrefix = "member." ) +// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): +// `member.`. Its end while the member is not built fails the dependent's step naming the member. +func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey } + +// MemberBoundOf is the build key a member wait's bound names, and whether key is one. +func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) } + // UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966, // novox/hq ADR 0668 §3): `unput.`. Its end while the key still stands so, with no look holding it, is the // finding call-unanswered about the key. @@ -68,7 +76,8 @@ const BoundsTool = "builder_bounds" const BoundsDescription = "The builder's expiring keys, its state " + BoundsState + " (design 51 *Health and conditions*): each key with what it " + "bounds (a version's publish, whose end unanswered is the finding " + PublishUnanswered + "; or a node's seven days holding the old " + "form's window, whose end is the finding " + OldFormHeld + "; or a build key left building because its end could not be written, " + - "whose end is the finding call-unanswered), when it ends, and how long is left. The bus ends each key by its own time to live." + "whose end is the finding call-unanswered; or a dependent's wait for a package member of a group's run, whose end fails its " + + "step), when it ends, and how long is left. The bus ends each key by its own time to live." // Bound is a key's value in BoundsState: what it bounds, in plain words, and when it ends. type Bound struct { From 260a16181c51dee08b516309138bbff19f19c9d7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:12:27 +0100 Subject: [PATCH 2/9] BL7: reserved by name from the catalog half's view, builder_versions calls it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's tools bundle serves reserved given a package's name: it follows the catalog half's packages view itself, finds the module from the name, answers the reserved version, or for a version no reservation holds asks the seat mesh-catalog's versions and answers released with the release's digests; a name no entry carries, two entries carry, or a version neither reserved nor released is answered error. builder_versions by name calls reserved through its invokes. The builder's process serves no call. Builds: design 47 G §572, §582 (A-70) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder-tools/main.go | 67 +++++++++++++++- .../mesh-delivery-builder-tools/reserved.go | 14 +++- .../reservedname_test.go | 58 ++++++++++++++ .../internal/versions/names.go | 54 ++++++++++--- .../internal/versions/reserve.go | 48 ++++++++++++ .../internal/versions/reservedname_test.go | 76 +++++++++++++++++++ .../internal/versions/tools.go | 5 ++ .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 1 + 9 files changed, 308 insertions(+), 17 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reservedname_test.go create mode 100644 modules/mesh-delivery-builder/internal/versions/reservedname_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index dea457ae..221653b6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -15,6 +15,8 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -35,8 +37,11 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error) // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for // the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer. +// +// builder_versions, asked by a name, answers also what the verb reserved answers for it (reserve; design 47 G §572): the +// bundle calls the seat mesh-builds through its invokes, as any caller does. Nil answers none. func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader, - fw, of versions.Reader) []stdio.Tool { + fw, of versions.Reader, reserve func(name, version string) (meshbuilds.ReservedAnswer, error)) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } // handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error. handAct := func(key string, value []byte, err error) (any, error) { @@ -91,7 +96,16 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q "version": map[string]any{"type": "string", "description": "one version, to ask whether it is recorded"}, "digest": map[string]any{"type": "string", "description": "with version: the digest the holder received"}}}, Run: func(args map[string]any) (any, error) { - return versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest")) + out, err := versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest")) + if err != nil || reserve == nil { + return out, err + } + ra, err := reserve(out.Name, str(args, "version")) + if err != nil { + return nil, fmt.Errorf("the seat mesh-builds' reserved did not answer for %s: %w", out.Name, err) + } + out.Reserved = &ra + return out, nil }}, {Name: build.StuckTool, Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " + @@ -314,6 +328,26 @@ func written(s stdio.KeptState, key string, revision uint64) time.Time { return time.Time{} } +// The state the bundle follows, its owner's tool, and the seat verbs it calls (the manifest's invokes). +const ( + indexerModule = "mesh-delivery-indexer" + packagesState = "packages" + packagesTool = indexerModule + ".indexer_read_packages" + catalogVersionsVerb = "seat:mesh-catalog.versions" + reservedVerb = "seat:mesh-builds.reserved" +) + +// followedPackages is the catalog half's packages state as the bundle's picture of it holds it (versions.PackagesState). +type followedPackages struct{ r *views.Reader } + +func (f followedPackages) Keys(context.Context) ([]string, error) { + return f.r.Keys(indexerModule + "." + packagesState) +} + +func (f followedPackages) Get(_ context.Context, key string) ([]byte, bool, error) { + return f.r.Get(indexerModule+"."+packagesState, key) +} + func main() { steps := map[string]build.StateRead{} for _, bm := range build.StuckBuildModules { @@ -370,8 +404,33 @@ func main() { } return e.Value, true, nil }} - tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) - tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + // The catalog half's view of the packages the bundle follows itself (design 47 G §572, A-70), as other bundles follow views. + pk := &views.Reader{Module: builds.BuilderModule, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, + Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} + pk.Follow(indexerModule, packagesState, packagesTool) + pk.Start(context.Background()) + carriers := versions.Carriers(followedPackages{pk}) + released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) { + raw, err := stdio.Ask(catalogVersionsVerb, meshcatalog.VersionsInput{Module: module}) + if err != nil { + return nil, err + } + var a meshcatalog.VersionsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the seat mesh-catalog's versions answered what does not read: %w", err) + } + return a.Versions, nil + } + reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) { + raw, err := stdio.Ask(reservedVerb, meshbuilds.ReservedInput{Name: name, Version: version}) + if err != nil { + return meshbuilds.ReservedAnswer{}, err + } + var a meshbuilds.ReservedAnswer + return a, json.Unmarshal(raw, &a) + } + tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve) + tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask}, carriers, released)...) ws := stdio.State(build.WaitsState) wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := ws.Get(key) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go index 8e3d342c..d6a45807 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "strings" @@ -16,7 +17,10 @@ import ( // reservedTools are builder_reserved, builder_reserved_drop and the seat mesh-builds' verb reserved (design 47 G section // 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to // reasks for the builder, which owns the reservations, to answer. -func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { +// +// The verb reserved given a name finds the module from the catalog half's view the bundle follows (carriers) and, for a version +// no reservation holds, asks the seat mesh-catalog's verb versions (released); the builder's process serves no call (A-70). +func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder, carriers func(context.Context, string) ([]versions.Built, error), released versions.CatalogVersions) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: versions.ReservedTool, Description: versions.ReservedDescription, @@ -25,10 +29,14 @@ func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, act Run: func(args map[string]any) (any, error) { return versions.AnswerReserved(rr, str(args, "module")) }}, {Name: versions.ReservedVerb, Description: "The seat mesh-builds' verb reserved: a module's reserved version with each artifact's digest once built " + "(design 47 G section 369), for a registry holder deciding whether to publish. Read-only.", - Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ - "module": map[string]any{"type": "string", "description": "the module"}, + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module; give this or name"}, + "name": map[string]any{"type": "string", "description": "a package's or an image's name in its ecosystem, in place of module"}, "version": map[string]any{"type": "string", "description": "the version, when asked for a particular one"}}}, Run: func(args map[string]any) (any, error) { + if name := str(args, "name"); name != "" { + return versions.AnswerReservedByName(context.Background(), rr, carriers, released, name, str(args, "version")) + } return versions.AnswerReservedVerb(rr, str(args, "module"), str(args, "version")) }}, {Name: versions.ReservedDropTool, Description: versions.ReservedDropDescription, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reservedname_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reservedname_test.go new file mode 100644 index 00000000..e23c2dba --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reservedname_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// Design 47 G §572, §582, A-70: the bundle's verb reserved takes a name and answers from the view it follows and the catalog's +// versions, and builder_versions, asked by a name, calls reserved and answers what it answers. +func TestReservedByNameIsTheBundlesAndBuilderVersionsCallsIt(t *testing.T) { + none := versions.Reader{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, bool, error) { return nil, false, nil }} + carriers := func(_ context.Context, name string) ([]versions.Built, error) { + if name == "@novox/mesh-sdk" { + return []versions.Built{{Module: "mesh-sdk", Artifact: "ts"}}, nil + } + return nil, nil + } + released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) { + return []meshcatalog.VersionEntry{{Module: module, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}}}, nil + } + acts := meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }} + byName := map[string]func(map[string]any) (any, error){} + for _, tl := range reservedTools(none, nil, time.Now, acts, carriers, released) { + byName[tl.Name] = tl.Run + } + got, err := byName[versions.ReservedVerb](map[string]any{"name": "@novox/mesh-sdk", "version": "0.1.2"}) + want := meshbuilds.ReservedAnswer{Module: "mesh-sdk", Version: "0.1.2", Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}} + if err != nil || got.(meshbuilds.ReservedAnswer).Released != want.Released || got.(meshbuilds.ReservedAnswer).Artifacts[0] != want.Artifacts[0] { + t.Fatalf("reserved by name = %+v (%v), want %+v", got, err, want) + } + var called []string + reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) { + called = append(called, name+"@"+version) + return want, nil + } + tools := Tools(build.StuckReader{}, nil, nil, nil, nil, time.Now, none, none, none, reserve) + for _, tl := range tools { + if tl.Name == versions.VersionsTool { + out, err := tl.Run(map[string]any{"name": "@novox/mesh-sdk", "version": "0.1.2"}) + ans, ok := out.(versions.VersionsAnswer) + if err != nil || !ok || ans.Reserved == nil || ans.Reserved.Module != "mesh-sdk" || !ans.Reserved.Released { + t.Fatalf("builder_versions by name = %+v (%v), want reserved's answer in it", out, err) + } + } + } + if len(called) != 1 || called[0] != "@novox/mesh-sdk@0.1.2" { + t.Errorf("reserved was called %v, want once, by the name and version", called) + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/names.go b/modules/mesh-delivery-builder/internal/versions/names.go index e6e7d091..bb1881c0 100644 --- a/modules/mesh-delivery-builder/internal/versions/names.go +++ b/modules/mesh-delivery-builder/internal/versions/names.go @@ -2,6 +2,8 @@ package versions import ( "context" + "errors" + "fmt" "sort" "strings" @@ -71,22 +73,28 @@ func isSeriesKey(key string) bool { return ok } -// ModuleOf is the module and artifact a package's name in its ecosystem is built from, as the indexer's packages state holds -// them (the inverse of NamesIn; design 47 G §373, §465): the series key whose value names that name. -func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { - return func(ctx context.Context, name string) (string, string, bool, error) { +// Built is a module's artifact, which a package's name is the name of. +type Built struct{ Module, Artifact string } + +// Carriers are the module artifacts a package's name in its ecosystem is the name of, as the catalog half's packages view +// holds them (the inverse of NamesIn; design 47 G §572, §478): each distinct module and artifact of a series key whose value +// names that name, sorted. A name no entry carries answers none. +func Carriers(s PackagesState) func(ctx context.Context, name string) ([]Built, error) { + return func(ctx context.Context, name string) ([]Built, error) { keys, err := s.Keys(ctx) if err != nil { - return "", "", false, err + return nil, err } sort.Strings(keys) + seen := map[Built]bool{} + var out []Built for _, key := range keys { if !isSeriesKey(key) { continue } value, found, err := s.Get(ctx, key) if err != nil { - return "", "", false, err + return nil, err } if !found { continue @@ -95,10 +103,38 @@ func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, a if err != nil || v.Name != name { continue } - if module, artifact, cut := strings.Cut(v.Package, "/"); cut && module != "" && artifact != "" { - return module, artifact, true, nil + if module, artifact, cut := strings.Cut(v.Package, "/"); cut && module != "" && artifact != "" && !seen[Built{module, artifact}] { + seen[Built{module, artifact}] = true + out = append(out, Built{module, artifact}) } } - return "", "", false, nil + return out, nil } } + +// ModuleOf is the module and artifact a package's name is built from (Carriers), found false for a name no entry carries; more +// than one entry carrying it is an error naming them. +func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { + carriers := Carriers(s) + return func(ctx context.Context, name string) (string, string, bool, error) { + got, err := carriers(ctx, name) + switch { + case err != nil: + return "", "", false, err + case len(got) == 0: + return "", "", false, nil + case len(got) > 1: + return "", "", false, errors.New(MoreThanOne(name, got)) + } + return got[0].Module, got[0].Artifact, true, nil + } +} + +// MoreThanOne is the words of a name that more than one entry carries. +func MoreThanOne(name string, got []Built) string { + var refs []string + for _, b := range got { + refs = append(refs, b.Module+"/"+b.Artifact) + } + return fmt.Sprintf("the name %s is carried by more than one entry: %s", name, strings.Join(refs, ", ")) +} diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 89b0dc32..9e190a6f 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -11,6 +11,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" ) // **The reservation is one key per module and MAJOR.MINOR** (design 47 G §369, §52): the builder keeps in its old state @@ -436,6 +437,53 @@ func AnswerReservedVerb(r Reader, module, version string) (meshbuilds.ReservedAn return out, nil } +// CatalogVersions answers a module's released versions, the seat mesh-catalog's verb versions (design 47 G §582). +type CatalogVersions func(ctx context.Context, module string) ([]meshcatalog.VersionEntry, error) + +// AnswerReservedByName is the verb reserved given a package's name (design 47 G §572, §582, A-70): the module is found from the +// name in the catalog half's view (carriers); that module's reserved version answers as AnswerReservedVerb does. A version +// given with no reservation of it (a reservation goes once its version is released) is asked of the catalog, and answers +// released with the artifacts and digests the release recorded. A name no entry carries, more than one entry carries, or a +// version neither reserved nor released is answered error, naming which, set alone. +func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.Context, string) ([]Built, error), released CatalogVersions, name, version string) (meshbuilds.ReservedAnswer, error) { + name, version = strings.TrimSpace(name), strings.TrimSpace(version) + got, err := carriers(ctx, name) + if err != nil { + return meshbuilds.ReservedAnswer{}, fmt.Errorf("the entries of %s cannot be read: %w", name, err) + } + switch { + case len(got) == 0: + return meshbuilds.ReservedAnswer{Error: "no entry carries the name " + name}, nil + case len(got) > 1: + return meshbuilds.ReservedAnswer{Error: MoreThanOne(name, got)}, nil + } + module := got[0].Module + rs := &Reserver{Store: readOnlyReserve{readStore{r}}} + if _, found, err := rs.Of(ctx, module, version); err != nil { + return meshbuilds.ReservedAnswer{}, err + } else if found { + return AnswerReservedVerb(r, module, version) + } + if version != "" { + entries, err := released(ctx, module) + if err != nil { + return meshbuilds.ReservedAnswer{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err) + } + for _, e := range entries { + if e.Version != version { + continue + } + out := meshbuilds.ReservedAnswer{Module: module, Version: version, Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}} + for _, a := range e.Artifacts { + out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a.Artifact, Digest: a.Digest}) + } + return out, nil + } + return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil + } + return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil +} + // The descriptions of the reservation's tools. const ( ReservedDescription = "The versions the builder reserved (its state reserved, design 47 G section 369): one per module and MAJOR.MINOR, " + diff --git a/modules/mesh-delivery-builder/internal/versions/reservedname_test.go b/modules/mesh-delivery-builder/internal/versions/reservedname_test.go new file mode 100644 index 00000000..ee5b48f2 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/reservedname_test.go @@ -0,0 +1,76 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// TestReservedByNameAnswersFromTheViewAndTheCatalog: design 47 G §572, §582, A-70 — a package's name (an npm name or a Go +// module path) is mapped to its module from the catalog half's view; the reserved version answers as for the module, a version +// no reservation holds is asked of the catalog's versions and answers released with the release's digests, and a name no entry +// carries, one more than one carries, or a version neither reserved nor released answers error, set alone. +func TestReservedByNameAnswersFromTheViewAndTheCatalog(t *testing.T) { + ctx := context.Background() + view := packagesMem{ + "git=2enovox=2ebe/novox/mesh-sdk/go.0.1": []byte(`{"v":1,"name":"git.novox.be/novox/mesh-sdk/go","package":"mesh-sdk/go","series":"0.1"}`), + "git=2enovox=2ebe/novox/mesh-sdk/go.0.2": []byte(`{"v":1,"name":"git.novox.be/novox/mesh-sdk/go","package":"mesh-sdk/go","series":"0.2"}`), + "@novox/mesh-sdk.0.1": []byte(`{"v":1,"name":"@novox/mesh-sdk","package":"mesh-sdk/ts","series":"0.1"}`), + "twice.0.1": []byte(`{"v":1,"name":"twice","package":"a/x","series":"0.1"}`), + "twice-b.0.1": []byte(`{"v":1,"name":"twice","package":"b/y","series":"0.1"}`), + } + store := &reserveMem{mem: newMem()} + r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.3"}, nil }} + if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{}); err != nil { + t.Fatal(err) + } + rd := Reader{ + Keys: func() ([]string, error) { return store.KeysOf(ctx, AllFilter) }, + Get: func(key string) (json.RawMessage, bool, error) { + raw, _, ok, err := store.Get(ctx, key) + return raw, ok, err + }} + var asked []string + released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) { + asked = append(asked, module) + return []meshcatalog.VersionEntry{{Module: module, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mesh-sdk:go", Digest: "sha256:dd"}}}}, nil + } + carriers := Carriers(view) + ask := func(name, version string) meshbuilds.ReservedAnswer { + t.Helper() + got, err := AnswerReservedByName(ctx, rd, carriers, released, name, version) + if err != nil { + t.Fatalf("%s %s: %v", name, version, err) + } + return got + } + if got := ask("@novox/mesh-sdk", ""); got.Module != "mesh-sdk" || got.Version != "0.1.4" || got.Released || got.Error != "" { + t.Errorf("an npm name = %+v, want the module's reserved version", got) + } + if got := ask("git.novox.be/novox/mesh-sdk/go", "0.1.2"); !got.Released || got.Module != "mesh-sdk" || len(got.Artifacts) != 1 || got.Artifacts[0].Digest != "sha256:dd" { + t.Errorf("a Go module path = %+v, want the release's artifacts and digests (two series of one entry are one entry)", got) + } + if len(asked) != 1 || asked[0] != "mesh-sdk" { + t.Errorf("the catalog was asked for %v, want the module's versions once, for the version no reservation holds", asked) + } + for name, c := range map[string]struct{ name, version, want string }{ + "unknown name": {"nothing", "", "no entry carries"}, + "two entries": {"twice", "", "more than one entry"}, + "neither released": {"git.novox.be/novox/mesh-sdk/go", "9.9.9", "reserved or released"}, + } { + if got := ask(c.name, c.version); !strings.Contains(got.Error, c.want) || got.Module != "" || len(got.Artifacts) != 0 { + t.Errorf("%s = %+v, want error naming %q alone", name, got, c.want) + } + } + if _, err := AnswerReservedByName(ctx, rd, Carriers(packagesMem(nil)), released, "x", ""); err == nil { + t.Error("an unreadable view was answered as if no entry carried the name") + } + if _, err := AnswerReservedByName(ctx, rd, carriers, func(context.Context, string) ([]meshcatalog.VersionEntry, error) { return nil, errors.New("down") }, "git.novox.be/novox/mesh-sdk/go", "0.1.2"); err == nil { + t.Error("an unreadable catalog was answered as if the version were not released") + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index 798bb0f8..a375b67d 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" ) // The builder's tools of its versions (design 51 *Events of the builder*, *Tools of the builder*). @@ -64,6 +66,9 @@ type VersionsAnswer struct { Asked string `json:"asked,omitempty"` Recorded *bool `json:"recorded,omitempty"` Why string `json:"why,omitempty"` + // Reserved is what the seat mesh-builds' verb reserved answers for the name (and version): the reserved or released version + // with each artifact's digest, or its error (design 47 G §572, §582). + Reserved *meshbuilds.ReservedAnswer `json:"reserved,omitempty"` } // AnswerVersions is builder_versions: a name's versions in order, in one call; with a version (and a digest), whether diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 835fd25d..0a6d1dc5 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -143,7 +143,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", - "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", + "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e43d5995..8995002a 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -82,6 +82,7 @@ "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", + "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", From 5524cfa8d807c755952758e90e1bacf97cb96196 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:27:01 +0100 Subject: [PATCH 3/9] BL6: the builder follows the catalog half per watched branch through the shared branch follower MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder adopts meshcatalogclient's branch follower on its views reader (A-5, A-15): the modules, their manifests (kept from the entry that holds them, by its digest), their packages and each package's name in its ecosystem are read from the catalog half's entries, never the indexer's modules, manifests, candidates and packages views; an entry of a branch other than the module's first builds its fingerprint and supersedes only its own branch's older one; the start's sweep runs once every followed half's first view is in; the builder's tools bundle reads package names through the same follower. A run's candidate is no input of the builder (A-84): its read of the indexer's candidates view goes, and the entry point for a run's build ask is not built (no line gives the ask's form). The seats' events the builder's own handler takes are named per seat so the follower's events have one taker. Builds: design 47 G §348, §479, §614, A-5, A-15, A-84 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder-tools/main.go | 25 +- .../cmd/mesh-delivery-builder/bus.go | 24 -- .../cmd/mesh-delivery-builder/catalog.go | 191 ++++++++++++++++ .../cmd/mesh-delivery-builder/catalog_test.go | 27 +++ .../catalogholder_integration_test.go | 94 ++++++++ .../mesh-delivery-builder/integration_test.go | 214 ++++++------------ .../cmd/mesh-delivery-builder/main.go | 116 +++------- .../cmd/mesh-delivery-builder/names.go | 28 ++- .../cmd/mesh-delivery-builder/names_test.go | 12 +- .../cmd/mesh-delivery-builder/versions.go | 49 ---- .../internal/build/entry_test.go | 28 +++ .../internal/build/run.go | 87 ++++++- .../internal/versions/entrynames.go | 82 +++++++ .../internal/versions/entrynames_test.go | 55 +++++ .../internal/versions/names.go | 6 +- .../mesh-delivery-builder/manifest_test.go | 16 +- modules/mesh-delivery-builder/module.json | 15 +- 17 files changed, 711 insertions(+), 358 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/entry_test.go create mode 100644 modules/mesh-delivery-builder/internal/versions/entrynames.go create mode 100644 modules/mesh-delivery-builder/internal/versions/entrynames_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 221653b6..29740838 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -15,6 +15,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -328,26 +329,12 @@ func written(s stdio.KeptState, key string, revision uint64) time.Time { return time.Time{} } -// The state the bundle follows, its owner's tool, and the seat verbs it calls (the manifest's invokes). +// The seat verbs the bundle calls (the manifest's invokes). const ( - indexerModule = "mesh-delivery-indexer" - packagesState = "packages" - packagesTool = indexerModule + ".indexer_read_packages" catalogVersionsVerb = "seat:mesh-catalog.versions" reservedVerb = "seat:mesh-builds.reserved" ) -// followedPackages is the catalog half's packages state as the bundle's picture of it holds it (versions.PackagesState). -type followedPackages struct{ r *views.Reader } - -func (f followedPackages) Keys(context.Context) ([]string, error) { - return f.r.Keys(indexerModule + "." + packagesState) -} - -func (f followedPackages) Get(_ context.Context, key string) ([]byte, bool, error) { - return f.r.Get(indexerModule+"."+packagesState, key) -} - func main() { steps := map[string]build.StateRead{} for _, bm := range build.StuckBuildModules { @@ -407,9 +394,13 @@ func main() { // The catalog half's view of the packages the bundle follows itself (design 47 G §572, A-70), as other bundles follow views. pk := &views.Reader{Module: builds.BuilderModule, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} - pk.Follow(indexerModule, packagesState, packagesTool) + branches, err := meshcatalogclient.FollowBranches(pk, stdio.Ask) + if err != nil { + fmt.Fprintln(os.Stderr, "mesh-delivery-builder-tools:", err) + os.Exit(1) + } pk.Start(context.Background()) - carriers := versions.Carriers(followedPackages{pk}) + carriers := versions.CarriersFrom(versions.EntriesOf(branches)) released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) { raw, err := stdio.Ask(catalogVersionsVerb, meshcatalog.VersionsInput{Module: module}) if err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go index 34654335..694d507d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go @@ -3,13 +3,9 @@ package main import ( "context" "errors" - "fmt" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "mesh-delivery-builder/internal/build" ) @@ -106,23 +102,3 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key } } } - -// manifestReader reads manifests. of the indexer and parses the manifest it holds. -type manifestReader struct{ kv jetstream.KeyValue } - -func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) { - key, err := meshcatalogclient.ManifestsKey(digest) - if err != nil { - return catalogue.Manifest{}, err - } - e, err := m.kv.Get(ctx, key) - if err != nil { - return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err) - } - kept, err := meshcatalogclient.DecodeManifest(e.Value()) - if err != nil { - return catalogue.Manifest{}, err - } - // Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside. - return catalogue.Decode([]byte(kept.Manifest)) -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go new file mode 100644 index 00000000..19515895 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -0,0 +1,191 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/views" + "mesh-delivery-builder/internal/build" +) + +// **The builder reads the catalog half per watched branch** (design 47 G §348, §479, §614; A-5, A-15): the shared branch follower +// of meshcatalogclient, adopted on the builder's views reader, gives each watched branch's entries; the manifests, the packages +// and each package's name in its ecosystem are read from the entries, never the indexer's states. + +// entryManifests keeps the manifest of each entry heard, by the entry's manifest digest (the entry holds the manifest whole). +type entryManifests struct { + mu sync.Mutex + byFp map[string]catalogue.Manifest +} + +// keep takes the manifest of e in; an entry with no manifest or digest keeps none. +func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error { + if e.ManifestDigest == "" || len(e.Manifest) == 0 { + return nil + } + raw, err := json.Marshal(e.Manifest) + if err != nil { + return err + } + // Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside. + mf, err := catalogue.Decode(raw) + if err != nil { + return fmt.Errorf("the manifest of the entry of %s does not decode: %w", e.Module, err) + } + m.mu.Lock() + defer m.mu.Unlock() + if m.byFp == nil { + m.byFp = map[string]catalogue.Manifest{} + } + m.byFp[e.ManifestDigest] = mf + return nil +} + +// Manifest is the manifest the entry heard with that digest holds. +func (m *entryManifests) Manifest(_ context.Context, digest string) (catalogue.Manifest, error) { + m.mu.Lock() + defer m.mu.Unlock() + mf, ok := m.byFp[digest] + if !ok { + return catalogue.Manifest{}, fmt.Errorf("no entry of the catalog half holds the manifest %s", digest) + } + return mf, nil +} + +// catalogHalf is the builder's part over the branch follower: what changes in a branch's half goes to the builder, and the +// entries of every followed half are what the package names are read from. +type catalogHalf struct { + branches *meshcatalogclient.Branches + b *build.Builder + manifests *entryManifests + logf func(string, ...any) + // named is, by module, the package names of its first entry last heard, so a name newly heard looks at its module again. + mu sync.Mutex + named map[string]string + // ready is told when every followed branch's first view is in, once; halves are the views adopted (sweepWiring). + ready func() + halves map[string]*views.Follower + once sync.Once + ctx context.Context +} + +// first says whether branch is the first watched branch of module's registration (design 47 G §371); a module with no +// registration heard is taken as watched on this branch, the first one. +func (c *catalogHalf) first(branch, module string) bool { + reg, ok := c.branches.Registrations.Registration(module) + return !ok || len(reg.Branches) == 0 || reg.Branches[0] == branch +} + +// Changed is the follower's Changed: one module's entry of a branch came, or went. +func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, entry json.RawMessage, deleted bool, at time.Time) { + return func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) { + first := c.first(branch, module) + if deleted { + c.b.EntryGone(branch, first, module) + return + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(entry, &e); err != nil || e.Module != module { + c.logf("the catalog half of the branch %q holds what is not the entry of %s, left: %v", branch, module, err) + return + } + if err := c.manifests.keep(e); err != nil { + c.logf("%v", err) + } + c.b.Entry(ctx, branch, first, e) + c.mu.Lock() + was := c.named[module] + now := namesOf(e) + if c.named == nil { + c.named = map[string]string{} + } + c.named[module] = now + c.mu.Unlock() + if first && now != "" && now != was { + c.logf("the catalog half's entry names a package of %s: its build is looked at again", module) + c.b.PackageNamed(ctx, module) + } + } +} + +// namesOf are the package names an entry's artifacts carry, in one line. +func namesOf(e meshcatalog.CatalogEntry) string { + var names []string + for _, a := range e.Artifacts { + if a.Package != nil && a.Package.Name != "" { + names = append(names, a.Ref+"="+a.Package.Name) + } + } + slices.Sort(names) + return fmt.Sprint(names) +} + +// sweepWiring is the follower's wiring on the builder's reader that learns each catalog half view adopted, so ready is told once +// when the first view of every branch followed is in (a branch with no revision yet is ready and empty at once). +type sweepWiring struct { + meshcatalogclient.Wiring + half *catalogHalf +} + +func (w sweepWiring) Adopt(f *views.Follower, seatName, event string, hear func(context.Context, []byte) error) error { + if err := w.Wiring.Adopt(f, seatName, event, hear); err != nil { + return err + } + if seatName == meshcatalog.Seat && event == meshcatalog.EventCatalogRevision { + w.half.adopted(f) + } + return nil +} + +func (w sweepWiring) Drop(owner, state string) error { + err := w.Wiring.Drop(owner, state) + w.half.dropped(state) + return err +} + +func (c *catalogHalf) adopted(f *views.Follower) { + c.mu.Lock() + if c.halves == nil { + c.halves = map[string]*views.Follower{} + } + c.halves[f.State] = f + c.mu.Unlock() + go func() { + select { + case <-c.ctx.Done(): + case <-f.Ready(): + c.checkReady() + } + }() +} + +func (c *catalogHalf) dropped(state string) { + c.mu.Lock() + delete(c.halves, state) + c.mu.Unlock() + c.checkReady() +} + +// checkReady calls ready once every adopted half's first view is in. +func (c *catalogHalf) checkReady() { + c.mu.Lock() + all := len(c.halves) > 0 + for _, f := range c.halves { + select { + case <-f.Ready(): + default: + all = false + } + } + c.mu.Unlock() + if all && c.ready != nil { + c.once.Do(c.ready) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go new file mode 100644 index 00000000..87768101 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest +// digest (design 47 G §348), and an entry with no manifest keeps none. +func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) { + m := &entryManifests{} + e := meshcatalog.CatalogEntry{Module: "netcheck", ManifestDigest: "sha256:aa", Manifest: map[string]any{"module": "netcheck", "version": "1"}} + if err := m.keep(e); err != nil { + t.Fatal(err) + } + if mf, err := m.Manifest(context.Background(), "sha256:aa"); err != nil || mf.Module != "netcheck" { + t.Errorf("Manifest(sha256:aa) = %+v, %v", mf, err) + } + if _, err := m.Manifest(context.Background(), "sha256:bb"); err == nil { + t.Error("Manifest of a digest no entry holds: want an error") + } + if err := m.keep(meshcatalog.CatalogEntry{Module: "bare"}); err != nil || len(m.byFp) != 1 { + t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go new file mode 100644 index 00000000..6d2950fe --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go @@ -0,0 +1,94 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" +) + +// catalogHolder is the holder of the seat mesh-catalog as the bed needs it (design 47 G §348, §614): it answers `revision` for the +// branch main with the entries put so far, `registrations` with none (so the follower follows the branches the gate's `source` +// answers) and `entry`, and says `catalog-revision` at each put, as the follower and the builder hear it. +type catalogHolder struct { + ctx context.Context + js jetstream.JetStream + + mu sync.Mutex + entries map[string]meshcatalog.CatalogEntry + revision int64 +} + +func newCatalogHolder(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder { + t.Helper() + h := &catalogHolder{ctx: ctx, js: js, entries: map[string]meshcatalog.CatalogEntry{}} + answer := func(subject string, with func(in []byte) any) { + s, err := nc.Subscribe(subject, func(m *nats.Msg) { + reply, _ := json.Marshal(map[string]any{"result": with(m.Data), "node": "ctl"}) + _ = m.Respond(reply) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Unsubscribe() }) + } + answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbRevision+".>", func([]byte) any { return h.whole() }) + answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbRegistrations+".>", func([]byte) any { + return meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}} + }) + answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbEntry+".>", func(in []byte) any { + var q meshcatalog.EntryInput + _ = json.Unmarshal(in, &q) + h.mu.Lock() + defer h.mu.Unlock() + for _, e := range h.entries { + if e.Fp == q.Fingerprint { + return meshcatalog.EntryAnswer{Entry: e} + } + } + return map[string]any{"error": "no entry of that fingerprint"} + }) + answer("mesh.seat."+meshdelivery.Seat+".call."+meshdelivery.VerbSource+".>", func([]byte) any { + return meshdelivery.SourceAnswer{Branches: []string{"main"}} + }) + return h +} + +// whole is the newest trunk revision of main, every entry in one page; the branch has none yet before the first put. +func (h *catalogHolder) whole() meshcatalog.RevisionAnswer { + h.mu.Lock() + defer h.mu.Unlock() + if h.revision == 0 { + return meshcatalog.RevisionAnswer{NoneYet: true} + } + var refs []meshcatalog.EntryRef + var all []meshcatalog.CatalogEntry + for m, e := range h.entries { + refs = append(refs, meshcatalog.EntryRef{Module: m, Digest: "sha256:" + e.Fp}) + all = append(all, e) + } + at := time.Now() + return meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: h.revision, Kind: "trunk", Branch: "main", Entries: refs, Source: 1, At: &at}, Entries: all} +} + +// put sets one module's entry in a new trunk revision of main and says catalog-revision. +func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { + t.Helper() + h.mu.Lock() + h.revision++ + h.entries[e.Module] = e + ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: []string{e.Module}, V: 1}) + h.mu.Unlock() + if _, err := h.js.Publish(h.ctx, meshcatalog.SubjectCatalogRevision("main"), ev); err != nil { + t.Fatal(err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 19619406..3d5831a7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -219,20 +219,18 @@ func (b *bed) trunk(t *testing.T, head string) uint64 { } type bed struct { - ctx context.Context - js jetstream.JetStream - modules jetstream.KeyValue - candidates jetstream.KeyValue - source jetstream.KeyValue - builds jetstream.KeyValue - heard jetstream.KeyValue // the state toolchains-heard - instances jetstream.KeyValue - packages jetstream.KeyValue // the indexer's packages, served by its read tool and views family - out *saidLog // the builder's process log, every instance's - stop func() // stops the builder started last - fake *fakeBuildModules - srvURL string - dir string + ctx context.Context + js jetstream.JetStream + catalog *catalogHolder // the seat mesh-catalog, as its holder answers the branch follower + source jetstream.KeyValue + builds jetstream.KeyValue + heard jetstream.KeyValue // the state toolchains-heard + instances jetstream.KeyValue + out *saidLog // the builder's process log, every instance's + stop func() // stops the builder started last + fake *fakeBuildModules + srvURL string + dir string } // startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the @@ -300,17 +298,12 @@ func startBedWith(t *testing.T, closed bool) *bed { } return kv } - b.modules = mk(indexerBucket(modulesState), modulesHistory) - b.candidates = mk(indexerBucket(candidatesState), candidatesHistory) b.source = mk(sourceBucket, sourceHistory) - mk(indexerBucket(manifestsState), manifestsHistory) - // The indexer and the assigner as the owners of the states the builder follows: their read tools and views families - // (ADR 0615, DLV-M1); n1 and n2 each have their durable consumer. + // The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's + // verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2 + // each have their durable consumer. + b.catalog = newCatalogHolder(t, ctx, js, ctl) readers := []string{"n1_" + builds.BuilderModule, "n2_" + builds.BuilderModule} - for _, st := range []string{modulesState, manifestsState, candidatesState} { - viewstest.Serve(t, ctx, js, ctl, indexerModule, st, indexerModule+".indexer_read_"+st, readers...) - } - b.packages = viewstest.Serve(t, ctx, js, ctl, indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages", readers...) viewstest.Serve(t, ctx, js, ctl, assignerModule, "assignments", assignerModule+".assigner_read_assignments", readers...) b.builds = mk(builds.BuilderBuildsBucket, builds.BuilderBuildsHistory) b.instances = mk(instancesBucket, 1) @@ -400,28 +393,34 @@ func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[ t.Fatal(err) } digest := meshcatalogclient.ManifestDigest(raw) - manifests, err := b.js.KeyValue(b.ctx, indexerBucket(manifestsState)) - if err != nil { - t.Fatal(err) - } - mv, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: name, Manifest: string(raw)}) - if err != nil { - t.Fatal(err) - } - if _, err := manifests.Put(b.ctx, digest, mv); err != nil { - t.Fatal(err) - } - data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: at, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2, - Manifest: digest, Path: path, Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)}) - if err != nil { - t.Fatal(err) - } - if _, err := b.modules.Put(b.ctx, name, data); err != nil { + var manifest map[string]any + if err := json.Unmarshal(raw, &manifest); err != nil { t.Fatal(err) } + b.catalog.put(t, b.entry(name, path, at, fp2, digest, manifest, toolchains, src)) return digest } +// manifestOf is a fixture manifest as the entry holds it. +func (b *bed) manifestOf(t *testing.T, fixture string) map[string]any { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", fixture)) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// entry is the catalog graph entry of a module as the holder answers it. +func (b *bed) entry(name, path, at, fp2, digest string, manifest map[string]any, toolchains map[string]builds.ToolchainUsed, src uint64) meshcatalog.CatalogEntry { + return meshcatalog.CatalogEntry{Module: name, Repository: "novox/mesh-catalog", Commit: at, Fp: fp2, Path: path, ManifestDigest: digest, Manifest: manifest, + Inputs: &meshcatalog.EntryInputs{Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)}} +} + // archive writes modules. as the real indexer does for a module of an archive only: its fingerprint names no // toolchain (mesh-delivery-indexer internal/index inputs.go languagesOf gives an archive no language), so Toolchains is empty. func (b *bed) archive(t *testing.T, name, path, at string) string { @@ -455,6 +454,12 @@ type saidLog struct { buf strings.Builder } +func (l *saidLog) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.String() +} + func (l *saidLog) Write(p []byte) (int, error) { l.mu.Lock() defer l.mu.Unlock() @@ -572,11 +577,12 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) } } - // Design 32, ADR 0615 §1: a package's name the indexer writes later reaches the builder by its packages' views family - // (never their bucket), and looks at the module again; the done bundles are kept, not numbered twice. - if _, err := b.packages.Put(b.ctx, "netcheck.0.1", []byte(`{"v":1,"name":"netcheck","package":"netcheck/lib","series":"0.1"}`)); err != nil { - t.Fatal(err) - } + // Design 32, design 47 G §348: a package's name the catalog half's entry of the module gives later reaches the builder by the + // branch follower's view (never the indexer's state), and looks at the module again; the done bundles are kept, not numbered twice. + named := b.entry("netcheck", "modules/netcheck", commit, fp, digest, nil, bothToolchains, 0) + named.Manifest = b.manifestOf(t, "catalogue-a67f0196-netcheck.module.json") + named.Artifacts = []meshcatalog.Artifact{{Ref: "netcheck:lib", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "netcheck"}}} + b.catalog.put(t, named) b.awaitSaid(t, "names a package of netcheck") // Design 47 G §349: version.numbered is no longer said, for a number or for a late name. if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { @@ -639,7 +645,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 reg.awaitFinding(t, failed, true) b.fake.hang.Store(nil) close(release) - b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) // the indexer writes it again + b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, "89abcdef0123456789abcdef0123456789abcdef", third, 3) // the entry moves to a later commit, same fingerprint b.awaitFP(t, "netcheck", third, isState(builds.BuildDone)) b.awaitSaid(t, "kept: fingerprint "+third+" is numbered under it") if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e builds.BuildLogEntry) bool { @@ -739,7 +745,7 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit) b.await(t, "netcheck2", isState(builds.BuildDone)) if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil { - t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2") + t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2; the log: " + b.out.String()) } if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 { t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n) @@ -778,77 +784,14 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) { } } -// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its -// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes -// builds.netcheck. done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source -// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its -// fingerprint first appeared at). -func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1 +// TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes: the catalog half's entry of imaged (one image +// artifact) is built on n1, where mesh-build-image runs: its key is done with the digest and the reference, its build call carrying +// publish; the entry's next fingerprint is built again with publish and ends with a reference of its own (ADR 0487 clauses 5 and +// 12; a run's candidate is no input of the builder: design 47 G A-84). +func TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes(t *testing.T) { // 1705 b := startBed(t) other := "src2:" + strings.Repeat("e", 64) - digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") - data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: other, - Manifest: digest, Path: "modules/netcheck", Toolchains: build.CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})}) - if err != nil { - t.Fatal(err) - } - if _, err := b.modules.Put(b.ctx, "netcheck", data); err != nil { - t.Fatal(err) - } - const head = "1111111111111111111111111111111111111111" - pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"netcheck"}, - Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"netcheck": {Fp2: fp, Manifest: digest}}}} - hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) - cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) - if err != nil { - t.Fatal(err) - } - ckey, err := keys.EscapeName("feat/x"), error(nil) - if err != nil { - t.Fatal(err) - } - if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil { - t.Fatal(err) - } - b.start(t, "n1", nil) - v := b.await(t, "netcheck", isState(builds.BuildDone)) - if len(v.Artifacts) != 2 { - t.Fatalf("build = %+v", v) - } - b.fake.mu.Lock() - defer b.fake.mu.Unlock() - found := false - for _, f := range b.fake.fetches { - if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead { - found = true - } - } - if !found { - t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches) - } -} - -// TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes: modules.imaged (one image artifact) is built on n1, where -// mesh-build-image runs: a trunk record's key is done with the digest and the reference, its build call carrying publish; a -// candidate's key for the same module is built first with no publish and no reference; when the pull request is merged and the -// trunk's record names the candidate's fingerprint, the key is built again with publish and ends with a reference -// (ADR 0487 clauses 5 and 12). -func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 1705 - b := startBed(t) - other := "src2:" + strings.Repeat("e", 64) - digest := b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit) - // The trunk carries `other`; its key is built and published. - trunk := func(fp2 string) { - data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2, - Manifest: digest, Path: "modules/imaged"}) - if err != nil { - t.Fatal(err) - } - if _, err := b.modules.Put(b.ctx, "imaged", data); err != nil { - t.Fatal(err) - } - } - trunk(other) + digest := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, other, 0) b.start(t, "n1", nil) okey, _ := builds.BuilderBuildKey("imaged", other) awaitKey := func(key string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { @@ -860,42 +803,23 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 } } } - t.Fatalf("builds.%s = %+v within 20 s", key, last) + t.Fatalf("builds.%s = %+v within 20 s; the log: %s", key, last, b.out.String()) return last } v := awaitKey(okey, isState(builds.BuildDone)) if len(v.Artifacts) != 1 || v.Artifacts[0].Digest != digestOf("server") || !strings.HasSuffix(v.Artifacts[0].Reference, "@"+digestOf("server")) { - t.Fatalf("trunk build = %+v, want the digest and the reference", v) + t.Fatalf("build = %+v, want the digest and the reference", v) } - // The candidate carries fp: built, not published. - const head = "1111111111111111111111111111111111111111" - pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"imaged"}, - Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"imaged": {Fp2: fp, Manifest: digest}}}} - hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}}) - cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) - if err != nil { - t.Fatal(err) + // The module's next fingerprint: built once more, published. + if got := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, fp, 0); got != digest { + t.Fatalf("manifest digest = %s, want %s", got, digest) } - ckey := keys.EscapeName("feat/x") - if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil { - t.Fatal(err) - } - v = b.await(t, "imaged", isState(builds.BuildDone)) - if len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { - t.Fatalf("candidate build = %+v, want done with no reference", v) - } - b.fake.mu.Lock() - if len(b.fake.builds) != 2 || b.fake.builds[0]["publish"] != true || b.fake.builds[1]["publish"] != nil { - t.Errorf("builds = %v, want the trunk's with publish, then the candidate's with none", b.fake.builds) - } - b.fake.mu.Unlock() - // The pull request is merged: the trunk's record names fp, and its key ends published, built once more with publish. - trunk(fp) - v = b.await(t, "imaged", func(v builds.BuilderBuild) bool { + fkey, _ := builds.BuilderBuildKey("imaged", fp) + v = awaitKey(fkey, func(v builds.BuilderBuild) bool { return v.State == builds.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Reference != "" }) if v.Artifacts[0].Digest != digestOf("server") { - t.Errorf("merged build = %+v", v) + t.Errorf("second build = %+v", v) } b.fake.mu.Lock() defer b.fake.mu.Unlock() @@ -905,8 +829,8 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170 published++ } } - if len(b.fake.builds) != 3 || published != 2 { - t.Errorf("builds = %v, want three, two of them with publish (the trunk's two fingerprints)", b.fake.builds) + if len(b.fake.builds) != 2 || published != 2 { + t.Errorf("builds = %v, want two, both with publish", b.fake.builds) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index adac30c3..6f2702b0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -30,10 +30,10 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/trunk" @@ -78,10 +78,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // The indexer's modules, manifests, candidates and packages and the assigner's assignments are their owners' states, read by the - // first view of their read tools and then their views families, never their buckets (ADR 0615 §1, DLV-M1). An owner that - // is down never holds the start: each is read on a goroutine once its first view is answered, and until then the - // unanswered first view is the builder's finding. + // The assigner's assignments are its state, read by the first view of its read tool and then its views family, never its + // bucket (ADR 0615 §1, DLV-M1); the catalog half is read through the seat mesh-catalog. An owner that is down never holds the + // start: each is read on a goroutine once its first view is answered, and until then the unanswered first view is the + // builder's finding. calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) if err != nil { return err @@ -105,24 +105,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } }) reader.Findings = func() views.WireWriter { return fw } - modules, err := reader.Follow(indexerModule, modulesState, indexerModule+".indexer_read_modules") - if err != nil { - return err - } - manifests, err := reader.Follow(indexerModule, manifestsState, indexerModule+".indexer_read_manifests") - if err != nil { - return err - } - // The candidates' fingerprints are built too (issue 1684). - candidates, err := reader.Follow(indexerModule, candidatesState, indexerModule+".indexer_read_candidates") - if err != nil { - return err - } - // The indexer's packages, whose names a package is numbered under (design 32). - packages, err := reader.Follow(indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages") + // The catalog half per watched branch (design 47 G §348, §479, §614; A-5, A-15): the shared branch follower of + // meshcatalogclient on the builder's reader, so the modules, their manifests, their packages and each package's name in its + // ecosystem are read from the entries, never the indexer's states. + half := &catalogHalf{manifests: &entryManifests{}, logf: logger.Printf, ctx: ctx} + branches, err := meshcatalogclient.NewBranches(sweepWiring{meshcatalogclient.ProcessWiring(reader), half}, calls.Call) if err != nil { return err } + branches.Logf = logger.Printf + half.branches = branches assignments, err := reader.Follow(assignerModule, "assignments", assignerModule+".assigner_read_assignments") if err != nil { return err @@ -171,9 +163,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Ready(func(kv jetstream.KeyValue) { floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf)) }) - // A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32). + // A package is numbered under its name in its ecosystem, read from the catalog half's entries (design 32, design 47 G §348). numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, - Names: versions.NamesIn(indexerPackages{packages})} + Names: versions.NamesFrom(versions.EntriesOf(branches))} // A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder // answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read. pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf) @@ -187,7 +179,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Gone: heldOnTheBus{js}, CheckAfter: bound, Toolchains: tools, - Manifests: manifestReader{manifests}, + Manifests: half.manifests, Logf: logger.Printf, Stored: storedVia(calls.Call), Node: cred.Node, @@ -245,7 +237,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro trunks.start() b.Trunk = trunks.head b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf} - b.Members = versions.ModuleOf(indexerPackages{packages}) // the build of a group's package member (design 47 G §373, §465) + b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the // indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since. b.Log = func(e builds.BuildLogEntry) { @@ -380,11 +372,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } heardGone(ctx, b, m, logger.Printf) } - // The seats' events (artifacts-gone, the registry seats' words, manifest-accepted and manifest-reads-changed and the - // seat mesh-conditions' condition-cleared, which this pattern covers: a consumer takes no two overlapping filters); the - // owners' views events are the views reader's. - if err := hub.Handle("mesh.seat.*.event.>", events); err != nil { - return err + // The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, manifest-accepted and + // manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the + // catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has + // one taker, so no pattern here covers them). + for _, pattern := range seatEventPatterns { + if err := hub.Handle(pattern, events); err != nil { + return err + } } hub.OnBound(reads.atBind) // what a node's engine said before the consumer was bound (ADR 0668 §4) if stopped != nil { @@ -397,10 +392,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return hub.Start(ctx, js, cred) }) go func() { <-ctx.Done(); hub.Stop() }() - // The modules, once their first views and the manifests' are answered: what a module builds is read from the manifests. - go func() { errs <- buildFromModules(ctx, b, modules, manifests, candidates) }() - // A package's name heard in the indexer's packages looks at its module again, so a done build is numbered then. - go func() { errs <- followPackages(ctx, packages, b, logger.Printf) }() + // The catalog half's entries as they come: a module's entry of a branch goes to the builder, and a key left building for a + // fingerprint no branch carries is closed (issue 1660) once the first view of every followed branch is in (never at the start: + // a holder of the seat that is down holds no other part of it). + half.b = b + half.ready = func() { + if err := b.Sweep(ctx); err != nil { + errs <- err + } + } + branches.Changed = half.Changed(ctx) // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as // every other input. The builder asks again by itself without it. // A state not on the bus yet at this start is watched once it appears (issue 1790). @@ -482,55 +483,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // keeper raises a first view unanswered past its bound (ADR 0615 §6). var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState -// The indexer's states the builder follows (indexerBucket(modulesState), ManifestsBucket, CandidatesBucket), and the owners' -// views families as their manifests' `streams` name them. -const ( - modulesState = "modules" - manifestsState = "manifests" - candidatesState = "candidates" -) - -// buildFromModules builds from the indexer's modules and candidates once the first views of the modules, the manifests and -// the candidates are answered (never at the start: an indexer that is down holds no other part of it), and follows them. -func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, candidates *views.Picture) error { - for _, p := range []*views.Picture{modules, manifests, candidates} { - select { - case <-ctx.Done(): - return nil - case <-p.Follower().Ready(): - } - } - each := func(e jetstream.KeyValueEntry) { - if opOf(e) { - b.Module(ctx, e.Key(), e.Value()) - } else { - b.Forget(e.Key()) - } - } - w, err := settle(ctx, modules, each) - if err != nil { - return err - } - // A key left building for a fingerprint its module no longer has is closed (issue 1660); the modules are all known now. - if err := b.Sweep(ctx); err != nil { - return err - } - // The fingerprints the indexer gives the candidates' modules are built too (issue 1684), once the trunk's modules are known. - eachCandidate := func(e jetstream.KeyValueEntry) { - if opOf(e) { - b.Candidate(ctx, e.Key(), e.Value()) - } else { - b.ForgetCandidate(e.Key()) - } - } - cw, err := settle(ctx, candidates, eachCandidate) - if err != nil { - return err - } - go func() { _ = follow(ctx, cw, eachCandidate) }() - return follow(ctx, w, each) -} - // sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest // sourceHistory of them: what a candidate's base revision names. type sourceHeads struct { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go index b52510ea..53c8b837 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go @@ -21,17 +21,27 @@ const ( // sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names. sourceBucket = gateModule + "_source" - // packagesBucket is the indexer's state packages. - packagesBucket = indexerModule + "_" + meshcatalog.StatePackages - // The revisions each keeps per key (the beds of the integration test create them alike). - sourceHistory = 64 - candidatesHistory = 10 - modulesHistory = 20 - manifestsHistory = 1 + // The revisions the gate's source keeps per key (the bed of the integration test creates it alike). + sourceHistory = 64 ) -func indexerBucket(state string) string { return indexerModule + "_" + state } - // namedTreePrefix opens the tree input of a module whose every artifact names its paths. const namedTreePrefix = "named:" + +// seatEventPatterns are the seats' events the builder's own handler takes off its durable consumer (its manifest's `consumes` of +// them): the registry seats' words, artifacts-gone, the build seats' toolchain-reported, condition-cleared, the node-engine +// seat's events and, of the seat mesh-catalog, manifest-accepted, version-released and versions-filled. The seat mesh-catalog's +// catalog-revision and registration-changed and the seat mesh-delivery's source-moved are the views reader's (the branch +// follower's), so no pattern here covers them: a message has one taker. +var seatEventPatterns = func() []string { + var out []string + for _, seat := range []string{"container-image-registry", "go-package-registry", "npm-package-registry", "oci-artifact-registry", + "mesh-artifact-sweep", "mesh-conditions", "node-engine", "node-build-go", "node-build-image", "node-build-node"} { + out = append(out, "mesh.seat."+seat+".event.>") + } + for _, event := range []string{meshcatalog.EventManifestAccepted, meshcatalog.EventVersionReleased, meshcatalog.EventVersionsFilled} { + out = append(out, "mesh.seat."+meshcatalog.Seat+".event."+event, "mesh.seat."+meshcatalog.Seat+".event."+event+".>") + } + return out +}() diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go index 2c1549b5..3b3efee3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "slices" + "strings" "testing" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -32,15 +33,20 @@ func TestTheNamesAreTheManifests(t *testing.T) { t.Errorf("the builder reads %s, its manifest's reads are %v", read, m.Reads) } } - for _, view := range []string{indexerModule + ".views.modules", indexerModule + ".views.manifests", indexerModule + ".views.candidates", - indexerModule + ".views.packages", assignerModule + ".views.assignments"} { + // The catalog half (design 47 G §348, §614) is followed through the seat, and no state of the indexer is. + for _, c := range m.Consumes { + if strings.HasPrefix(c, indexerModule+".") { + t.Errorf("the builder follows %s, an indexer state, its manifest consumes it", c) + } + } + for _, view := range []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", assignerModule + ".views.assignments"} { if !slices.Contains(m.Consumes, view) { t.Errorf("the builder follows %s, its manifest consumes %v", view, m.Consumes) } } for want, got := range map[string]string{ "mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), - buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-indexer_packages": packagesBucket, + buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket, } { if want != got { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index ed1460ab..24565e05 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -61,55 +61,6 @@ func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, b return e.Value(), e.Revision(), true, nil } -// indexerPackages is the indexer's packages state as the builder reads a package's name in its ecosystem from it (design -// 32): the picture of its read tool's first view and its views family (ADR 0615 §1), never its bucket. Until the first -// view is answered a package is built unnumbered and said (versions.Numberer.Names), and numbered once its name is heard -// (followPackages). -type indexerPackages struct{ kv jetstream.KeyValue } - -func (s indexerPackages) Keys(ctx context.Context) ([]string, error) { - keys, err := s.kv.Keys(ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil - } - return keys, err -} - -func (s indexerPackages) Get(ctx context.Context, key string) ([]byte, bool, error) { - e, err := s.kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) { - return nil, false, nil - } - if err != nil { - return nil, false, err - } - return e.Value(), true, nil -} - -// followPackages watches the picture of the indexer's packages state: each series key heard that names a package's name -// (versions.NamedModule), its first view's included, looks at its module again (build.Builder.PackageNamed), so a done -// build that left the package unnumbered is numbered then with its digests. Woken by the views family, never a timer; a -// key heard again naming the name and module it named (the indexer writes a series at each publish) looks at nothing. -func followPackages(ctx context.Context, kv jetstream.KeyValue, b *build.Builder, logf func(string, ...any)) error { - w, err := kv.WatchAll(ctx) - if err != nil { - return err - } - heard := map[string]string{} - return follow(ctx, w, func(e jetstream.KeyValueEntry) { - if e.Operation() != jetstream.KeyValuePut { - delete(heard, e.Key()) - return - } - module, name, named := versions.NamedModule(e.Key(), e.Value()) - if named && heard[e.Key()] != module+" "+name { - heard[e.Key()] = module + " " + name - logf("%s.%s names a package of %s: its build is looked at again", packagesBucket, e.Key(), module) - b.PackageNamed(ctx, module) - } - }) -} - func versionsConflict(err error) error { if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { return versions.ErrConflict diff --git a/modules/mesh-delivery-builder/internal/build/entry_test.go b/modules/mesh-delivery-builder/internal/build/entry_test.go new file mode 100644 index 00000000..0eaa2008 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/entry_test.go @@ -0,0 +1,28 @@ +package build + +import ( + "reflect" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy: a catalog graph entry gives the builder its fingerprint, manifest digest, +// source, the toolchains its fingerprint was computed with, its bases, packages, needs and the named paths of its artifacts by path +// (design 47 G §348, §614). +func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) { + e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f", + ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"}, + Inputs: &meshcatalog.EntryInputs{Src: 7, Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}}, + Paths: map[string][]string{"lib": {"a/b", "c"}}}} + got := ModuleOf(e) + if got.Fp2 != "src2:f" || got.Manifest != "sha256:m" || got.Path != "modules/netcheck" || got.Commit != "abc" || got.Src != 7 || + got.Toolchains["go"].Version != "go1" || got.Packages["p"] != "1" || !reflect.DeepEqual(got.Needs, []string{"n"}) || + got.Inputs2 == nil || !reflect.DeepEqual(got.Inputs2.Bases, []string{"base"}) || + !reflect.DeepEqual(got.Inputs2.Paths, map[string]string{"a/b": "lib", "c": "lib"}) { + t.Errorf("ModuleOf = %+v", got) + } + if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" { + t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 52402be3..f6f5574b 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -117,7 +117,8 @@ type Builder struct { mu sync.Mutex // fillWaits are the builds that said wait-held for the fill (module.fingerprint), so it is said once. fillWaits sync.Map - known map[string]meshcatalog.ModulesByModuleValue // by modules key + known map[string]meshcatalog.ModulesByModuleValue // by modules key: the first watched branch's + heard map[string]meshcatalog.ModulesByModuleValue // by branch and module (take): what supersedes what inflight map[string]bool // by build key again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built plans map[string]planned // by manifest digest @@ -169,7 +170,8 @@ func (b *Builder) init() { } b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} - b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} + b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{} + b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands = map[string]*handWatch{} b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{} @@ -186,8 +188,9 @@ func (b *Builder) logf(format string, args ...any) { // Wait returns when every build started so far has ended. func (b *Builder) Wait() { b.wg.Wait() } -// Module takes one value of the indexer's modules bucket in (the key is the module's name) and builds it if it is -// to be built. A value that is not a module is said and left. +// Module takes one value of a modules bucket in (the key is the module's name) and builds it if it is to be built. A value that +// is not a module is said and left. The builder takes its modules from the catalog half (Entry); this is the same take of one +// module's value, with no branch, the first one's. func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.init() m, err := meshcatalogclient.DecodeModule(value) @@ -195,9 +198,41 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.logf("modules.%s holds what is not a module, left: %v", key, err) return } + b.take(ctx, "", true, key, m) +} + +// Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its +// fingerprint if it is to be built. first says the branch is the first of the module's registration, whose entry is the module's +// value for what the builder reads by name (a version, a build again); an entry of another branch builds its fingerprint and +// supersedes only that branch's older one. An entry naming no module is said and left. +func (b *Builder) Entry(ctx context.Context, branch string, first bool, e meshcatalog.CatalogEntry) { + b.init() + if e.Module == "" { + b.logf("the catalog half of the branch %q holds an entry that names no module, left", branch) + return + } + b.take(ctx, branch, first, e.Module, ModuleOf(e)) +} + +// EntryGone takes a module's entry out of a branch's half (the module was dropped from it, or the branch is followed no more). +func (b *Builder) EntryGone(branch string, first bool, module string) { + b.init() b.mu.Lock() - old, had := b.known[key] - b.known[key] = m + delete(b.heard, branch+"\x00"+module) + b.mu.Unlock() + if first { + b.Forget(module) + } +} + +func (b *Builder) take(ctx context.Context, branch string, first bool, key string, m meshcatalog.ModulesByModuleValue) { + b.mu.Lock() + hk := branch + "\x00" + key + old, had := b.heard[hk] + b.heard[hk] = m + if first { + b.known[key] = m + } var superseded context.CancelCauseFunc if had && old.Fp2 != "" && old.Fp2 != m.Fp2 { if ok, err := builds.BuilderBuildKey(key, old.Fp2); err == nil { @@ -205,7 +240,7 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { } } if bk, err := builds.BuilderBuildKey(key, m.Fp2); err == nil && m.Fp2 != "" { - // A fingerprint the trunk carries is built from the trunk's commit, whoever asked for it first. + // A fingerprint a watched branch carries is built from that branch's commit, whoever asked for it first. b.trunkFP[bk] = true delete(b.via, bk) } @@ -218,6 +253,32 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.consider(ctx, key, m) } +// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository, +// directory, commit, the source revision it was taken in at), the toolchains its fingerprint was computed with, the bases it is +// built against, its packages and needs, and the named paths of its artifacts. +func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { + m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None, + Manifest: e.ManifestDigest, Packages: e.Packages, Needs: e.Needs} + if len(e.Bases) > 0 { + m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases} + } + if in := e.Inputs; in != nil { + m.Src, m.Toolchains, m.Problems = in.Src, in.Toolchains, in.Problems + if len(in.Paths) > 0 { + if m.Inputs2 == nil { + m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{} + } + m.Inputs2.Paths = map[string]string{} + for artifact, paths := range in.Paths { + for _, p := range paths { + m.Inputs2.Paths[p] = artifact + } + } + } + } + return m +} + // NodesChanged is Reconsider after a toolchain moved, so a build that waited for the nodes to agree on one may go on. func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) } @@ -234,10 +295,14 @@ func (b *Builder) Sweep(ctx context.Context) error { } b.mu.Lock() current := map[string]bool{} - for name, m := range b.known { - if m.Fp2 != "" { - if k, err := builds.BuilderBuildKey(name, m.Fp2); err == nil { - current[k] = true + // What every followed branch carries now is current, not only the first branch's (design 47 G §348). + for _, held := range []map[string]meshcatalog.ModulesByModuleValue{b.known, b.heard} { + for hk, m := range held { + name := hk[strings.LastIndex(hk, "\x00")+1:] + if m.Fp2 != "" { + if k, err := builds.BuilderBuildKey(name, m.Fp2); err == nil { + current[k] = true + } } } } diff --git a/modules/mesh-delivery-builder/internal/versions/entrynames.go b/modules/mesh-delivery-builder/internal/versions/entrynames.go new file mode 100644 index 00000000..a140613f --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/entrynames.go @@ -0,0 +1,82 @@ +package versions + +import ( + "context" + "sort" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Entries are the catalog graph entries of the halves the builder follows, one per module and watched branch (design 47 G §348, +// §614): where a package's name in its ecosystem is read from, each artifact's `package` (§676), never the indexer's state. +type Entries func() []meshcatalog.CatalogEntry + +// artifactOf is the artifact name of an artifact reference `:` of module. +func artifactOf(module, ref string) (string, bool) { + m, a, ok := strings.Cut(ref, ":") + return a, ok && m == module && a != "" +} + +// NamesFrom answers a package's name in its ecosystem from the entries (design 47 G §348): the name the entry of module gives the +// artifact; known is false when no entry names one for it. +func NamesFrom(all Entries) PackageNames { + return func(_ context.Context, module, artifact string) (string, bool, error) { + for _, e := range all() { + if e.Module != module { + continue + } + for _, a := range e.Artifacts { + if name, ok := artifactOf(module, a.Ref); ok && name == artifact && a.Package != nil && a.Package.Name != "" { + return a.Package.Name, true, nil + } + } + } + return "", false, nil + } +} + +// CarriersFrom are the module artifacts a package's name in its ecosystem is the name of, as the entries hold them (design 47 G +// §572, §478), each distinct module and artifact once, sorted; a name no entry carries answers none. +func CarriersFrom(all Entries) func(ctx context.Context, name string) ([]Built, error) { + return func(_ context.Context, name string) ([]Built, error) { + seen := map[Built]bool{} + var out []Built + for _, e := range all() { + for _, a := range e.Artifacts { + artifact, ok := artifactOf(e.Module, a.Ref) + if !ok || a.Package == nil || a.Package.Name != name || seen[Built{e.Module, artifact}] { + continue + } + seen[Built{e.Module, artifact}] = true + out = append(out, Built{e.Module, artifact}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Module+"/"+out[i].Artifact < out[j].Module+"/"+out[j].Artifact }) + return out, nil + } +} + +// ModuleFrom is the module and artifact a package's name is built from, from the entries (CarriersFrom); more than one entry +// carrying it is an error naming them. +func ModuleFrom(all Entries) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { + return moduleOfCarriers(CarriersFrom(all)) +} + +// EntriesOf are the entries of every half the branch follower holds, the followed branches in the order they came. +func EntriesOf(branches *meshcatalogclient.Branches) Entries { + return func() []meshcatalog.CatalogEntry { + var out []meshcatalog.CatalogEntry + for _, br := range branches.Followed() { + if cat, ok := branches.Catalog(br); ok { + if half, ok := cat.View(); ok { + for _, e := range half.Entries { + out = append(out, e) + } + } + } + } + return out + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/entrynames_test.go b/modules/mesh-delivery-builder/internal/versions/entrynames_test.go new file mode 100644 index 00000000..32097c78 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/entrynames_test.go @@ -0,0 +1,55 @@ +package versions + +import ( + "context" + "reflect" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +func named(module, artifact, name string) meshcatalog.CatalogEntry { + return meshcatalog.CatalogEntry{Module: module, Artifacts: []meshcatalog.Artifact{ + {Ref: module + ":" + artifact, Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: name}}}} +} + +// TestPackageNamesAreReadFromTheCatalogHalfsEntries: a package's name in its ecosystem is the name the entry of its module gives the +// artifact (design 47 G §348, §676), the module and artifact a name is built from are the entries carrying it, and a name two +// entries carry is an error naming both (§491); no state of the indexer is read. +func TestPackageNamesAreReadFromTheCatalogHalfsEntries(t *testing.T) { + entries := func() []meshcatalog.CatalogEntry { + return []meshcatalog.CatalogEntry{named("mesh-sdk", "go", "git.novox.be/novox/mesh-sdk/go"), named("netcheck", "lib", "netcheck"), + named("twin-a", "lib", "twin"), named("twin-b", "lib", "twin"), {Module: "bare"}} + } + ctx := context.Background() + for _, c := range []struct { + module, artifact string + name string + known bool + }{ + {"mesh-sdk", "go", "git.novox.be/novox/mesh-sdk/go", true}, + {"netcheck", "lib", "netcheck", true}, + {"netcheck", "other", "", false}, + {"bare", "lib", "", false}, + {"unknown", "lib", "", false}, + } { + got, known, err := NamesFrom(entries)(ctx, c.module, c.artifact) + if err != nil || got != c.name || known != c.known { + t.Errorf("NamesFrom(%s/%s) = %q, %v, %v; want %q, %v", c.module, c.artifact, got, known, err, c.name, c.known) + } + } + module, artifact, found, err := ModuleFrom(entries)(ctx, "netcheck") + if module != "netcheck" || artifact != "lib" || !found || err != nil { + t.Errorf("ModuleFrom(netcheck) = %q, %q, %v, %v", module, artifact, found, err) + } + if _, _, found, err := ModuleFrom(entries)(ctx, "nobody"); found || err != nil { + t.Errorf("ModuleFrom(nobody) = %v, %v; want none", found, err) + } + if _, _, _, err := ModuleFrom(entries)(ctx, "twin"); err == nil { + t.Error("ModuleFrom(twin): two entries carry it, want an error") + } + got, _ := CarriersFrom(entries)(ctx, "twin") + if want := []Built{{"twin-a", "lib"}, {"twin-b", "lib"}}; !reflect.DeepEqual(got, want) { + t.Errorf("CarriersFrom(twin) = %v, want %v", got, want) + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/names.go b/modules/mesh-delivery-builder/internal/versions/names.go index bb1881c0..a4b7e796 100644 --- a/modules/mesh-delivery-builder/internal/versions/names.go +++ b/modules/mesh-delivery-builder/internal/versions/names.go @@ -115,7 +115,11 @@ func Carriers(s PackagesState) func(ctx context.Context, name string) ([]Built, // ModuleOf is the module and artifact a package's name is built from (Carriers), found false for a name no entry carries; more // than one entry carrying it is an error naming them. func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { - carriers := Carriers(s) + return moduleOfCarriers(Carriers(s)) +} + +// moduleOfCarriers is ModuleOf over one way of finding the carriers. +func moduleOfCarriers(carriers func(ctx context.Context, name string) ([]Built, error)) func(ctx context.Context, name string) (module, artifact string, found bool, err error) { return func(ctx context.Context, name string) (string, string, bool, error) { got, err := carriers(ctx, name) switch { diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 0a6d1dc5..075fe81e 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -128,9 +128,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. // 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955). - // DLV-M1 (ADR 0615 §1): the indexer's modules, manifests and candidates and the assigner's assignments are asked by their - // read tools and followed by their views families, never read; so are the indexer's packages, whose names a package is - // numbered under (design 32). + // DLV-M1 (ADR 0615 §1): the assigner's assignments are asked by their read tool and followed by their views family, never + // read; the catalog half (the modules, their manifests, their packages and the package names a package is numbered under) is + // read through the seat mesh-catalog's branch follower (design 47 G §348, §614), and the indexer's states not at all. for _, r := range m.Reads { // The toolchain gate follows the assigner's assignments through meshmoduleassignerclient (buildrun.AssignmentsBucket), so // it declares that one read, as the console and route-proxy do; everything else of theirs is asked by tools. @@ -140,11 +140,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). - if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", - "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", + if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", - "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no @@ -157,10 +156,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed", "mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", - "mesh-delivery-indexer.views.packages", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", + "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", - "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), sorted(want)) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 8995002a..6f0808a6 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -68,10 +68,9 @@ "oci-artifact-registry.refused", "oci-artifact-registry.removed", "mesh-delivery-assigner.views.assignments", - "mesh-delivery-indexer.views.candidates", - "mesh-delivery-indexer.views.manifests", - "mesh-delivery-indexer.views.modules", - "mesh-delivery-indexer.views.packages" + "mesh-catalog.catalog-revision", + "mesh-catalog.registration-changed", + "mesh-delivery.source-moved" ], "invokes": [ "mesh-delivery-forge-bridge.bridge_tags", @@ -95,10 +94,10 @@ "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions", "mesh-delivery-assigner.assigner_read_assignments", - "mesh-delivery-indexer.indexer_read_candidates", - "mesh-delivery-indexer.indexer_read_manifests", - "mesh-delivery-indexer.indexer_read_modules", - "mesh-delivery-indexer.indexer_read_packages" + "seat:mesh-catalog.entry", + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", + "seat:mesh-delivery.source" ], "state": [ { "name": "builds", "history": 5 }, From 088f55f94cb742cb9ad7db74439e048af25c0eca Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:28:11 +0100 Subject: [PATCH 4/9] BL5 (watched-branch half): a build carries the branch and source revision of its catalog revision, and only the first watched branch reserves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An entry the builder takes from the catalog half carries its origin: the watched branch, the source revision of the catalog revision it came from and whether the branch is the first of the module's registration. A build of another branch reserves no version and is built and delivered by its fingerprint; a reservation of the first branch keeps the branch and the source revision for the publication. Builds: design 47 G §371, A-57 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/catalog.go | 8 +++- .../internal/build/numbers.go | 4 +- .../internal/build/reserve_test.go | 38 +++++++++++++++- .../internal/build/run.go | 45 ++++++++++++++----- .../internal/versions/reserve.go | 12 ++++- 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go index 19515895..cf2693f4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -91,6 +91,12 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e c.b.EntryGone(branch, first, module) return } + origin := build.Origin{Branch: branch, First: first} + if cat, ok := c.branches.Catalog(branch); ok { + if half, ok := cat.View(); ok { + origin.Source = half.Revision.Source + } + } var e meshcatalog.CatalogEntry if err := json.Unmarshal(entry, &e); err != nil || e.Module != module { c.logf("the catalog half of the branch %q holds what is not the entry of %s, left: %v", branch, module, err) @@ -99,7 +105,7 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e if err := c.manifests.keep(e); err != nil { c.logf("%v", err) } - c.b.Entry(ctx, branch, first, e) + c.b.Entry(ctx, origin, e) c.mu.Lock() was := c.named[module] now := namesOf(e) diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 26dd0fe5..1729a15a 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -172,12 +172,12 @@ func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bo // reserve reserves the version of the build that starts; a reservation that cannot be made is said and the build goes on // unreserved, as an unnumbered build does. -func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan) (versions.ReservedVersion, bool) { +func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, o Origin) (versions.ReservedVersion, bool) { refs := make([]string, 0, len(plan.Tasks)) for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Branch: o.Branch, Revision: o.Source} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index ae1627f1..c1b85a65 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -3,11 +3,13 @@ package build import ( "context" "fmt" + "reflect" "slices" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -21,12 +23,13 @@ type fakeReservations struct { reserved [][]string series []string built map[string]string + sources []versions.Source } func (f *fakeReservations) Filled() bool { return f.filled } -func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, _ versions.Source) (versions.ReservedVersion, error) { - f.reserved, f.series = append(f.reserved, artifacts), append(f.series, series) +func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) { + f.reserved, f.series, f.sources = append(f.reserved, artifacts), append(f.series, series), append(f.sources, src) return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil } @@ -127,3 +130,34 @@ func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) { } } } + +// TestOnlyABuildOfTheFirstWatchedBranchReservesAndItCarriesItsCatalogRevision: design 47 G §371, A-57 — a build of the registration's +// first watched branch reserves a version and the reservation carries the branch and the source revision of the catalog revision its +// entry came from; a build of another watched branch reserves none and is built and done. +func TestOnlyABuildOfTheFirstWatchedBranchReservesAndItCarriesItsCatalogRevision(t *testing.T) { + for _, c := range []struct { + name string + origin Origin + want []versions.Source + }{ + {"the first branch", Origin{Branch: "main", Source: 41, First: true}, + []versions.Source{{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Branch: "main", Revision: 41}}}, + {"another branch", Origin{Branch: "beta", Source: 42}, nil}, + } { + t.Run(c.name, func(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{filled: true} + r.b.Reserved = res + r.b.Entry(context.Background(), c.origin, meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog", Path: "modules/m", + Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1, ManifestDigest: mdigest('a'), Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)}}) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone { + t.Fatalf("build = %+v, want done", v) + } + if !reflect.DeepEqual(res.sources, c.want) { + t.Errorf("reserved from %+v, want %+v", res.sources, c.want) + } + }) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index f6f5574b..75bc44e7 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -119,6 +119,7 @@ type Builder struct { fillWaits sync.Map known map[string]meshcatalog.ModulesByModuleValue // by modules key: the first watched branch's heard map[string]meshcatalog.ModulesByModuleValue // by branch and module (take): what supersedes what + origin map[string]Origin // by build key: the catalog revision its entry came from (take) inflight map[string]bool // by build key again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built plans map[string]planned // by manifest digest @@ -171,6 +172,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{} + b.origin = map[string]Origin{} b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands = map[string]*handWatch{} @@ -198,20 +200,28 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.logf("modules.%s holds what is not a module, left: %v", key, err) return } - b.take(ctx, "", true, key, m) + b.take(ctx, Origin{First: true}, key, m) +} + +// Origin is the catalog revision an entry came from (design 47 G §371, A-57): the watched branch it is of, as named, the source +// revision the revision was taken in from, and whether the branch is the first of the module's registration, the only one whose +// builds reserve a version and whose entry is the module's value for what the builder reads by name (a version, a build again). +type Origin struct { + Branch string + Source int64 + First bool } // Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its -// fingerprint if it is to be built. first says the branch is the first of the module's registration, whose entry is the module's -// value for what the builder reads by name (a version, a build again); an entry of another branch builds its fingerprint and +// fingerprint if it is to be built. An entry of a branch other than the first builds its fingerprint, reserves none, and // supersedes only that branch's older one. An entry naming no module is said and left. -func (b *Builder) Entry(ctx context.Context, branch string, first bool, e meshcatalog.CatalogEntry) { +func (b *Builder) Entry(ctx context.Context, o Origin, e meshcatalog.CatalogEntry) { b.init() if e.Module == "" { - b.logf("the catalog half of the branch %q holds an entry that names no module, left", branch) + b.logf("the catalog half of the branch %q holds an entry that names no module, left", o.Branch) return } - b.take(ctx, branch, first, e.Module, ModuleOf(e)) + b.take(ctx, o, e.Module, ModuleOf(e)) } // EntryGone takes a module's entry out of a branch's half (the module was dropped from it, or the branch is followed no more). @@ -225,12 +235,12 @@ func (b *Builder) EntryGone(branch string, first bool, module string) { } } -func (b *Builder) take(ctx context.Context, branch string, first bool, key string, m meshcatalog.ModulesByModuleValue) { +func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue) { b.mu.Lock() - hk := branch + "\x00" + key + hk := o.Branch + "\x00" + key old, had := b.heard[hk] b.heard[hk] = m - if first { + if o.First { b.known[key] = m } var superseded context.CancelCauseFunc @@ -242,6 +252,7 @@ func (b *Builder) take(ctx context.Context, branch string, first bool, key strin if bk, err := builds.BuilderBuildKey(key, m.Fp2); err == nil && m.Fp2 != "" { // A fingerprint a watched branch carries is built from that branch's commit, whoever asked for it first. b.trunkFP[bk] = true + b.origin[bk] = o delete(b.via, bk) } b.mu.Unlock() @@ -279,6 +290,17 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { return m } +// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build +// of the trunk's value was before branches were watched. +func (b *Builder) originOf(key string) Origin { + b.mu.Lock() + defer b.mu.Unlock() + if o, ok := b.origin[key]; ok { + return o + } + return Origin{First: true} +} + // NodesChanged is Reconsider after a toolchain moved, so a build that waited for the nodes to agree on one may go on. func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) } @@ -583,7 +605,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return // a package member of the group is not built (design 47 G §373): its end looks at this key again } } - reserving := trunk && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) + // Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371). + reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) if reserving && b.waitFill(module, m) { return } @@ -624,7 +647,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB var reserved versions.ReservedVersion hasReserved := false if reserving && cl.Build.State != builds.BuildDone { - reserved, hasReserved = b.reserve(ctx, module, m, plan) + reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key)) } var result builds.BuilderBuild if numbering.Refused != "" && cl.Build.State != builds.BuildDone { diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 9e190a6f..3db6a5e6 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -54,7 +54,11 @@ type ReservedVersion struct { // package's, GoTagOf): empty for a module with no Go package (design 47 G §370). Repository string `json:"repository,omitempty"` Commit string `json:"commit,omitempty"` - Tag string `json:"tag,omitempty"` + // Branch and Source are the watched branch and the source revision of the catalog revision whose entry the build came from + // (design 47 G §371, A-57): what `package-published` says of the publication. + Branch string `json:"branch,omitempty"` + Source int64 `json:"source,omitempty"` + Tag string `json:"tag,omitempty"` } // Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is @@ -62,6 +66,10 @@ type ReservedVersion struct { type Source struct { Repository, Commit string GoPackage bool + // Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of + // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. + Branch string + Revision int64 } // ReservedKey is the key of module's reservation of series (MAJOR.MINOR). @@ -136,7 +144,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), - Repository: src.Repository, Commit: src.Commit} + Repository: src.Repository, Commit: src.Commit, Branch: src.Branch, Source: src.Revision} if src.GoPackage { out.Tag = GoTagOf(out.Version) } From 8ea78f709a9bccea378a0cc65da4dcf095d6ae51 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:30:45 +0100 Subject: [PATCH 5/9] BL1: the whole package-published per artifact on the registry seat's published, from the reservation and the step result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a registry seat says published, the builder says the seat mesh-builds' package-published once per artifact the reservation newly marks: the artifact reference, version and digest published, the watched branch and the source revision kept on the reservation from the entry the build came from (A-57), and the dependencies of kind 1 (resolved) and 3 (toolchain) the step result named (§466, §673), their parts the artifact references the entry's edges name. The reservation keeps the dependencies from the build's end until the publication. Builds: design 47 G §52, §334, §349, §371, §466, §673, A-57 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/main.go | 4 + .../cmd/mesh-delivery-builder/reserve.go | 19 +++- .../cmd/mesh-delivery-builder/reserve_test.go | 13 ++- .../internal/build/numbers.go | 6 +- .../internal/build/reserve_test.go | 45 +++++++- .../internal/build/run.go | 105 ++++++++++++++++-- .../internal/versions/reserve.go | 41 ++++++- .../internal/versions/reserve_test.go | 20 +++- 8 files changed, 223 insertions(+), 30 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 6f2702b0..9a5c7818 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -209,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // and version-released heard; nothing is reserved before the fill. reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf) reserved := newReservations(awaitedReserved{awaitedVersions{b: reservedKV, bucket: versions.ReservedBucket}}, calls.Call, pubBoundsKV.KV, found, b.Reconsider, logger.Printf) + // Each artifact the holder says published at a reserved version is said as the seat mesh-builds' package-published (design 47 G §52). + reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error { + return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p) + } b.Reserved = reserved // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 7d286318..ee2812f6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "strings" "sync/atomic" "time" @@ -48,6 +49,8 @@ type reservations struct { logf func(string, ...any) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller + // say says package-published on the seat mesh-builds; nil says none. + say func(context.Context, meshbuilds.PackagePublished) error } // newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's @@ -82,9 +85,9 @@ func (rv *reservations) Reserve(ctx context.Context, module, series string, arti // Built records the digests; once every artifact has one, the publish's bound starts (a key of the state bounds with its // own time to live): its end with the reservation standing is version-unreleased. -func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string) error { +func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error { for ref, d := range digests { - if _, err := rv.r.Built(ctx, module, version, ref, d); err != nil { + if _, err := rv.r.Built(ctx, module, version, ref, d, deps[ref]); err != nil { return err } } @@ -198,9 +201,19 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { // published marks the artifacts of the reservation a registry seat's published names (its version and digest). func (rv *reservations) published(ctx context.Context, o versions.Outcome) { - if _, err := rv.r.Published(ctx, o.Version, o.Digest); err != nil { + marked, err := rv.r.Published(ctx, o.Version, o.Digest) + if err != nil { rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err) } + // Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334). + for _, m := range marked { + if rv.say == nil { + continue + } + if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil { + rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err) + } + } } // dropped is builder_reserved_drop: the reservation is deleted and its bound ended. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index dca519a1..24a349b5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "mesh-delivery-builder/internal/versions" @@ -93,11 +94,13 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { if !rv.heard(ctx, filled) || !filled.acked || !rv.Filled() || woke != 1 { t.Fatalf("versions-filled: acked %v, filled %v, woke %d", filled.acked, rv.Filled(), woke) } - res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{}) + var said []meshbuilds.PackagePublished + rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil } + res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41}) if err != nil || res.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err) } - if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}); err != nil { + if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil { t.Fatal(err) } boundKey := versions.ReservedBoundKey("m", "1.4") @@ -105,6 +108,12 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { t.Fatalf("the publish's bound is not armed: %v", bounds.keys) } rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}) + // Design 47 G §52, A-57: the holder's published is said once, per artifact, as the seat mesh-builds' package-published with the + // branch and source revision of the reservation; a second published of the same artifact says nothing. + rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}) + if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != "1.4.3" || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 { + t.Fatalf("package-published said %+v, want one of m:a at 1.4.3 for main at source 41", said) + } if !rv.ended(ctx, purged{key: boundKey}) { t.Fatal("a reservation's bound was not taken in") } diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 1729a15a..6fd808cb 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -3,6 +3,7 @@ package build import ( "context" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-sdk/go/catalogue" "strconv" "strings" @@ -135,8 +136,9 @@ type Reservations interface { Filled() bool // Reserve reserves module's next version in series for the artifacts (their references, :). Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) - // Built records the digests of the artifacts built under the reserved version, by reference. - Built(ctx context.Context, module, version string, digests map[string]string) error + // Built records the digests of the artifacts built under the reserved version, by reference, and what each resolved and was + // built with (design 47 G §466, §673). + Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error } // reserves says whether a look at a trunk build is one that reserves: the build has a series of the form MAJOR.MINOR diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index c1b85a65..63b40d4d 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -2,6 +2,7 @@ package build import ( "context" + "encoding/json" "fmt" "reflect" "slices" @@ -9,6 +10,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -24,6 +26,7 @@ type fakeReservations struct { series []string built map[string]string sources []versions.Source + deps map[string][]meshbuilds.Dependency } func (f *fakeReservations) Filled() bool { return f.filled } @@ -33,7 +36,8 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil } -func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string) error { +func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error { + f.deps = deps f.built = map[string]string{version: ""} for ref, d := range digests { f.built[ref] = d @@ -161,3 +165,42 @@ func TestOnlyABuildOfTheFirstWatchedBranchReservesAndItCarriesItsCatalogRevision }) } } + +// TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith: design 47 G §466, §673, A-57 — the `resolved` of each artifact and the +// toolchain digest of the step's result are kept on the reservation as dependencies of kind 1 and 3, their parts the artifact +// references the entry's edges name; a package no edge names is left out and said. +func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{filled: true} + r.b.Reserved = res + var journal []string + r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) } + r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { + st := r.answer(a) + var out buildstep.BuildAnswer + if err := json.Unmarshal(st.Result, &out); err != nil || len(out.Artifacts) == 0 { + return st + } + out.Artifacts[0].Resolved = []buildstep.AskResolved{{Package: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.14", Digest: "sha256:11"}, + {Package: "unnamed/pkg", Version: "1.0.0", Digest: "sha256:22"}} + out.Toolchain = &buildstep.ToolchainUsed{Language: "go", Version: "go1.27.1", Recipe: "1", Digest: "sha256:tc"} + st.Result, _ = json.Marshal(out) + return st + } + r.b.Entry(context.Background(), Origin{Branch: "main", Source: 41, First: true}, meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog", + Path: "modules/m", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1, ManifestDigest: mdigest('a'), + Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)}, + Edges: []meshcatalog.Edge{{Kind: 1, From: "mesh-sdk:go", To: "m:tool", Why: "git.novox.be/novox/mesh-sdk/go"}, {Kind: 3, From: "go-toolchain:program", To: "m:tool"}}}) + r.b.Wait() + want := map[string][]meshbuilds.Dependency{"m:tool": {{Kind: 1, Part: "mesh-sdk:go", Version: "0.1.14", Digest: "sha256:11"}, + {Kind: 3, Part: "go-toolchain:program", Version: "go1.27.1", Digest: "sha256:tc"}}} + if !reflect.DeepEqual(res.deps, want) { + t.Errorf("dependencies = %+v, want %+v", res.deps, want) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "unnamed/pkg") && strings.Contains(l, "names no artifact") + }) { + t.Errorf("journal %q, want the package no edge names said", journal) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 75bc44e7..b0e4e4c7 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -1,6 +1,7 @@ package build import ( + "cmp" "context" "encoding/json" "errors" @@ -15,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -120,6 +122,8 @@ type Builder struct { known map[string]meshcatalog.ModulesByModuleValue // by modules key: the first watched branch's heard map[string]meshcatalog.ModulesByModuleValue // by branch and module (take): what supersedes what origin map[string]Origin // by build key: the catalog revision its entry came from (take) + edges map[string][]meshcatalog.Edge // by build key: the edges of its entry (take), whose parts a package-published names + deps map[string]map[string][]meshbuilds.Dependency // by build key: what each artifact resolved and was built with (keepDependencies) inflight map[string]bool // by build key again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built plans map[string]planned // by manifest digest @@ -172,7 +176,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{} - b.origin = map[string]Origin{} + b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{} b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} b.hands = map[string]*handWatch{} @@ -200,7 +204,7 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.logf("modules.%s holds what is not a module, left: %v", key, err) return } - b.take(ctx, Origin{First: true}, key, m) + b.take(ctx, Origin{First: true}, key, m, nil) } // Origin is the catalog revision an entry came from (design 47 G §371, A-57): the watched branch it is of, as named, the source @@ -221,7 +225,7 @@ func (b *Builder) Entry(ctx context.Context, o Origin, e meshcatalog.CatalogEntr b.logf("the catalog half of the branch %q holds an entry that names no module, left", o.Branch) return } - b.take(ctx, o, e.Module, ModuleOf(e)) + b.take(ctx, o, e.Module, ModuleOf(e), e.Edges) } // EntryGone takes a module's entry out of a branch's half (the module was dropped from it, or the branch is followed no more). @@ -235,7 +239,7 @@ func (b *Builder) EntryGone(branch string, first bool, module string) { } } -func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue) { +func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue, e []meshcatalog.Edge) { b.mu.Lock() hk := o.Branch + "\x00" + key old, had := b.heard[hk] @@ -253,6 +257,7 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog. // A fingerprint a watched branch carries is built from that branch's commit, whoever asked for it first. b.trunkFP[bk] = true b.origin[bk] = o + b.edges[bk] = e delete(b.via, bk) } b.mu.Unlock() @@ -716,7 +721,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB for _, a := range result.Artifacts { digests[module+":"+a.Name] = a.Digest } - if err := b.Reserved.Built(ctx, module, reserved.Version, digests); err != nil { + if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil { b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err) } } @@ -763,11 +768,14 @@ type lane struct { version string // the version the build reserved, MAJOR.MINOR.PATCH, empty for a build that reserves none (design 47 G §573) tasks []Task toolchain *builds.ToolchainUsed - digests map[string]string // artifact -> digest - references map[string]string // artifact -> the reference an image was pushed to (a trunk build only) - imports []string // the mesh packages the lane's programs import, as its build module answered them - kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty - outOfStep []string // the generated files its build module's compare found out of step, passed or failed + digests map[string]string // artifact -> digest + references map[string]string // artifact -> the reference an image was pushed to (a trunk build only) + imports []string // the mesh packages the lane's programs import, as its build module answered them + resolved map[string][]sdkstep.AskResolved // artifact -> the mesh packages its build resolved (design 47 G §466) + tcDigest string // the digest of the toolchain program the building node ran, kind 3's (design 47 G §466, §472) + tcVersion string // and the version it ran + kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty + outOfStep []string // the generated files its build module's compare found out of step, passed or failed err error } @@ -816,7 +824,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module } } // One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3). - lanes = append(lanes, &lane{module: bm, version: version, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}}) + lanes = append(lanes, &lane{module: bm, version: version, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}, resolved: map[string][]sdkstep.AskResolved{}}) } if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { b.handLanes(key, len(lanes)) @@ -829,6 +837,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module wg.Wait() if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil { b.keepOutOfStep(key, lanes) + b.keepDependencies(key, module, lanes) } var errs []string for _, l := range lanes { @@ -1138,6 +1147,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } l.imports = answer.Imports l.kept = answer.Kept + if answer.Toolchain != nil { + l.tcDigest, l.tcVersion = answer.Toolchain.Digest, answer.Toolchain.Version + } built := map[string]sdkstep.AskBuilt{} for _, a := range answer.Artifacts { built[a.Artifact] = a @@ -1149,6 +1161,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes return } l.digests[t.Artifact] = a.Digest + l.resolved[t.Artifact] = a.Resolved if a.Reference != "" && (stored(t.Kind) || !candidate) { l.references[t.Artifact] = a.Reference } @@ -1283,3 +1296,73 @@ func SittingOutWords(language string, a sdkstep.Agreement) []string { } return out } + +// keepDependencies keeps, for a build key, what its step results say each artifact resolved (kind 1) and was built with (kind 3) as +// the dependencies `package-published` carries (design 47 G §466, §673, A-57): the part of a kind-1 item is the artifact reference of +// the package, and of the kind-3 item the toolchain's part, each as the edges of the entry name it. A part no edge names is said and +// its item left out. +func (b *Builder) keepDependencies(key, module string, lanes []*lane) { + b.mu.Lock() + edges := b.edges[key] + b.mu.Unlock() + out := map[string][]meshbuilds.Dependency{} + for _, l := range lanes { + for _, t := range l.tasks { + ref := module + ":" + t.Artifact + var deps []meshbuilds.Dependency + for _, r := range l.resolved[t.Artifact] { + part := "" + for _, e := range edges { + if e.Kind == 1 && e.To == ref && e.Why == r.Package { + part = cmp.Or(e.Part, e.From) + } + } + if part == "" && b.Members != nil { + if mod, art, ok, err := b.Members(context.Background(), r.Package); ok && err == nil { + part = mod + ":" + art + } + } + if part == "" { + b.logf("%s: the package %s the build of %s resolved names no artifact: its dependency is not said", key, r.Package, ref) + continue + } + deps = append(deps, meshbuilds.Dependency{Kind: 1, Part: part, Version: r.Version, Digest: r.Digest}) + } + if l.tcDigest != "" { + part := "" + for _, e := range edges { + if e.Kind == 3 && e.To == ref { + part = cmp.Or(e.Part, e.From) + } + } + if part == "" { + b.logf("%s: no edge of kind 3 names the toolchain part the build of %s used: its dependency is not said", key, ref) + } else { + deps = append(deps, meshbuilds.Dependency{Kind: 3, Part: part, Version: l.tcVersion, Digest: l.tcDigest}) + } + } + if len(deps) > 0 { + out[ref] = deps + } + } + } + b.mu.Lock() + defer b.mu.Unlock() + if b.deps == nil { + b.deps = map[string]map[string][]meshbuilds.Dependency{} + } + if len(out) == 0 { + delete(b.deps, key) + return + } + b.deps[key] = out +} + +// takeDependencies answers the dependencies held for a build key and forgets them. +func (b *Builder) takeDependencies(key string) map[string][]meshbuilds.Dependency { + b.mu.Lock() + defer b.mu.Unlock() + d := b.deps[key] + delete(b.deps, key) + return d +} diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 3db6a5e6..c135f0d4 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -58,7 +58,10 @@ type ReservedVersion struct { // (design 47 G §371, A-57): what `package-published` says of the publication. Branch string `json:"branch,omitempty"` Source int64 `json:"source,omitempty"` - Tag string `json:"tag,omitempty"` + // Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with + // (kind 3), kept for `package-published` (design 47 G §466, §673, A-57). + Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"` + Tag string `json:"tag,omitempty"` } // Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is @@ -181,7 +184,7 @@ func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64 // Built records artifact's digest under the reserved version of module's series; once every artifact has one the build is // ended (BuiltAt). A reservation moved on to another version (a newer build took the next) is left alone and answers false. -func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string) (bool, error) { +func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string, deps []meshbuilds.Dependency) (bool, error) { series, _, err := SplitVersion(version) if err != nil { return false, err @@ -193,6 +196,12 @@ func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest return false, err } cur.Artifacts[artifact] = digest + if len(deps) > 0 { + if cur.Dependencies == nil { + cur.Dependencies = map[string][]meshbuilds.Dependency{} + } + cur.Dependencies[artifact] = deps + } if cur.BuiltAt.IsZero() && allSet(cur.Artifacts) { cur.BuiltAt = r.now() } @@ -313,9 +322,24 @@ func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error return true, r.Store.Delete(ctx, key) } +// Marked is an artifact of a reservation a registry seat's published newly marked: the reservation as it stands then. +type Marked struct { + Reserved ReservedVersion + Artifact string +} + +// PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's +// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and +// the source revision the build was made from, and the dependencies its step result named. +func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished { + at = at.UTC() + return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module, + Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]} +} + // Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form) -// whose built digest it names are marked published. It answers the keys it marked. -func (r *Reserver) Published(ctx context.Context, version, digest string) ([]string, error) { +// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked. +func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) { version = strings.TrimPrefix(version, "v") if version == "" || digest == "" { return nil, nil @@ -325,7 +349,7 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]str return nil, fmt.Errorf("the reservations cannot be listed: %w", err) } sort.Strings(keys) - var marked []string + var marked []Marked for _, k := range keys { again: cur, rev, found, err := r.get(ctx, k) @@ -336,14 +360,17 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]str continue } changed := false + var arts []string for a, d := range cur.Artifacts { if d == digest && !cur.Published[a] { if cur.Published == nil { cur.Published = map[string]bool{} } cur.Published[a], changed = true, true + arts = append(arts, a) } } + sort.Strings(arts) if !changed { continue } @@ -353,7 +380,9 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]str } else if err != nil { return marked, fmt.Errorf("%s cannot be written: %w", k, err) } - marked = append(marked, k) + for _, a := range arts { + marked = append(marked, Marked{Reserved: cur, Artifact: a}) + } } return marked, nil } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index cfecc500..49ab3a06 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -78,11 +78,11 @@ func TestReservation(t *testing.T) { // The artifacts' digests, then the bound. for _, a := range arts { - if ok, err := r.Built(ctx, "m", "0.1.23", a, "sha256:"+a); err != nil || !ok { + if ok, err := r.Built(ctx, "m", "0.1.23", a, "sha256:"+a, nil); err != nil || !ok { t.Fatalf("built %s: %v %v", a, ok, err) } } - if ok, _ := r.Built(ctx, "m", "0.1.22", "m:a", "x"); ok { + if ok, _ := r.Built(ctx, "m", "0.1.22", "m:a", "x", nil); ok { t.Error("a digest was written under a version a newer build superseded") } published := func(_, _, a string) bool { return a == "m:a" } @@ -118,11 +118,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { ctx := context.Background() store := &reserveMem{mem: newMem()} r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }} - v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{}) + v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41}) if err != nil || v.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err) } - if _, err := r.Built(ctx, "m", "1.4.3", "m:a", "sha256:aa"); err != nil { + deps := []meshbuilds.Dependency{{Kind: 1, Part: "mesh-sdk:go", Version: "0.1.14", Digest: "sha256:dd"}, {Kind: 3, Part: "go-toolchain:program", Version: "go1.27", Digest: "sha256:tt"}} + if _, err := r.Built(ctx, "m", "1.4.3", "m:a", "sha256:aa", deps); err != nil { t.Fatal(err) } rd := Reader{ @@ -142,9 +143,18 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 { t.Errorf("builder_reserved of another module = %v (%v), want none", got, err) } - if marked, err := r.Published(ctx, "v1.4.3", "sha256:aa"); err != nil || len(marked) != 1 { + marked, err := r.Published(ctx, "v1.4.3", "sha256:aa") + if err != nil || len(marked) != 1 { t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err) } + // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source + // revision the reservation was made for, and the dependencies its step result named. + at := time.Unix(100, 0) + said := marked[0].PackagePublished("sha256:aa", at) + atUTC := at.UTC() + if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { + t.Errorf("package-published = %+v, want %+v", said, wantSaid) + } if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 { t.Errorf("a published with another digest marked %v", marked) } From 84bcdb687a7d45f10eeb582f14df0d3cdb2c13c4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 00:04:34 +0100 Subject: [PATCH 6/9] BL18: build modules found through their build seats' holders on the seat list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder finds a build module as the indexer's seed does: the build seat whose acting rule `build` has an `on` matching the artifact's language or kind:image, and that seat's holder, read from the seat list's definitions and the claims of the catalog half's manifests. Task.Module, every lane key, the unpublished modules, the toolchain sources, the ends watched, the sayer and builder_stuck (through `.steps`) derive from it; the floor's sources are the holders of the seats git and mesh-catalog. buildrun.GoBuildModule, NodeBuildModule, LanguageOf, buildstep.BuildModuleOf use, build.ImageBuildModule and StuckBuildModules are gone from the builder (the checker keeps its own names in two small files until its part reads them from the list). Builds: design 47 G §7, §428, §704, §204, A-82 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-builder/buildrun/asks.go | 5 +- .../buildrun/asks_integration_test.go | 2 +- .../buildrun/asks_test.go | 42 ++--- .../buildrun/buildseats.go | 166 ++++++++++++++++++ .../buildrun/buildseats_test.go | 60 +++++++ .../buildrun/heardends.go | 30 +++- .../mesh-delivery-builder/buildrun/modules.go | 7 - .../buildrun/seats_test.go | 22 +++ .../buildrun/standing_test.go | 6 +- .../buildrun/toolchaingate_test.go | 64 +++---- .../buildrun/toolchains.go | 11 +- .../cmd/mesh-delivery-builder-tools/main.go | 22 ++- .../cmd/mesh-delivery-builder/catalog.go | 5 + .../catalogholder_integration_test.go | 50 ++++++ .../cmd/mesh-delivery-builder/held.go | 11 +- .../mesh-delivery-builder/integration_test.go | 4 +- .../cmd/mesh-delivery-builder/main.go | 50 ++++-- .../cmd/mesh-delivery-builder/names_test.go | 2 +- .../cmd/mesh-delivery-builder/refused_test.go | 3 +- .../cmd/mesh-delivery-builder/says.go | 6 +- .../cmd/mesh-delivery-builder/says_test.go | 2 +- .../cmd/mesh-delivery-builder/seatlist.go | 26 +++ .../cmd/mesh-delivery-builder/seats_test.go | 23 +++ .../cmd/mesh-delivery-builder/toolchains.go | 6 +- .../mesh-delivery-builder/toolchains_test.go | 10 +- .../cmd/mesh-delivery-builder/versions.go | 26 ++- .../internal/build/candidates.go | 3 +- .../internal/build/claim.go | 26 ++- .../internal/build/plan.go | 48 +++-- .../internal/build/plan_test.go | 57 ++++-- .../internal/build/round4_test.go | 10 +- .../internal/build/run.go | 34 ++-- .../internal/build/run_test.go | 18 +- .../internal/build/seats_test.go | 23 +++ .../internal/build/stuck.go | 28 ++- .../internal/build/waits.go | 3 +- .../internal/versions/entrynames.go | 55 ++++++ .../internal/versions/floor.go | 21 ++- .../internal/versions/versions_test.go | 3 +- .../mesh-delivery-builder/manifest_test.go | 10 +- modules/mesh-delivery-builder/module.json | 11 +- .../cmd/mesh-delivery-checker/buildmodules.go | 23 +++ .../cmd/mesh-delivery-checker/judge.go | 6 +- .../cmd/mesh-delivery-checker/judge_test.go | 7 +- .../cmd/mesh-delivery-checker/main.go | 12 +- .../cmd/mesh-delivery-checker/refused_test.go | 2 +- .../cmd/mesh-delivery-checker/toolchains.go | 8 +- .../internal/check/build_change.go | 5 +- .../internal/check/buildmodules.go | 11 ++ .../internal/check/buildseats_test.go | 16 ++ .../internal/check/checker_test.go | 2 +- .../internal/check/examiner_test.go | 18 +- .../internal/check/judge.go | 16 +- .../internal/check/lostask_test.go | 16 +- .../internal/check/pins_test.go | 3 +- .../rules/buildmodules.go | 8 + modules/mesh-delivery-checker/rules/facts.go | 3 +- modules/mesh-delivery-checker/rules/graph.go | 4 +- modules/mesh-delivery-checker/rules/judged.go | 3 +- modules/mesh-delivery-checker/rules/words.go | 3 +- 60 files changed, 903 insertions(+), 274 deletions(-) create mode 100644 modules/mesh-delivery-builder/buildrun/buildseats.go create mode 100644 modules/mesh-delivery-builder/buildrun/buildseats_test.go delete mode 100644 modules/mesh-delivery-builder/buildrun/modules.go create mode 100644 modules/mesh-delivery-builder/buildrun/seats_test.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seatlist.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/seats_test.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/buildmodules.go create mode 100644 modules/mesh-delivery-checker/internal/check/buildmodules.go create mode 100644 modules/mesh-delivery-checker/internal/check/buildseats_test.go create mode 100644 modules/mesh-delivery-checker/rules/buildmodules.go diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index bbc793fb..0a1c5825 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -36,6 +36,8 @@ type AskWaiter interface { type Asker struct { Publisher AskPublisher Waiter AskWaiter + // Seats finds the build module of an ask's language (design 47 G A-82). + Seats *BuildSeats // Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase, // before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault // (ADR 0525 §6, §9), and the bound starts again when another node takes the ask. @@ -142,10 +144,11 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { // putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node // that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx. func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error) { - module, err := buildstep.BuildModuleOf(a.Language) + bm, err := k.Seats.OfAskLanguage(a.Language) if err != nil { return buildstep.Step{}, err } + module := bm.Module watch, cancel := context.WithCancel(ctx) defer cancel() steps, err := k.Waiter.WatchAsk(watch, module, a.ID) diff --git a/modules/mesh-delivery-builder/buildrun/asks_integration_test.go b/modules/mesh-delivery-builder/buildrun/asks_integration_test.go index 31363fad..eb012857 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_integration_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_integration_test.go @@ -34,7 +34,7 @@ func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) { t.Fatal(err) } bus := JetStreamAsks{JS: js} - k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second} + k := Asker{Seats: testSeats(), Publisher: bus, Waiter: bus, Bound: 5 * time.Second} a := goAsk("mvl0frv18d1960") for i := 0; i < 2; i++ { if err := k.Put(ctx, a); err != nil { diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index ed127a56..95b0ff31 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -31,7 +31,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[]}`)} }} subjects := &subjectsOf{inner: f} - k := Asker{Publisher: subjects, Waiter: f} + k := Asker{Seats: testSeats(), Publisher: subjects, Waiter: f} var seen, also []buildstep.Step // issue 1890: a waiter that asks to see each value of the step sees it // Issue 1921: a second seen on the same context (a check's begin bound inside a try) is added, never in place of the first. both := WithStepSeen(WithStepSeen(context.Background(), func(s buildstep.Step) { seen = append(seen, s) }), @@ -68,7 +68,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { if c.transport { restore = busretry.Waits(20*time.Millisecond, time.Millisecond, time.Millisecond) // a bound this row passes } - err := (Asker{Publisher: p, Waiter: p.inner}).Put(context.Background(), goAsk("mvl0frv18d1960")) + err := (Asker{Seats: testSeats(), Publisher: p, Waiter: p.inner}).Put(context.Background(), goAsk("mvl0frv18d1960")) restore() switch { case c.transport: @@ -94,7 +94,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { for _, st := range order { h.Write(st) } - if got, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1971")); err != nil || got.State != buildstep.StepPassed { + if got, err := (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1971")); err != nil || got.State != buildstep.StepPassed { t.Errorf("%s: %+v, %v", name, got, err) } } @@ -106,12 +106,12 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { } h.Write(st) } - if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") { + if _, err := (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") { t.Errorf("three runs then failed: %v", err) } h = &FakeAsks{} h.Write(buildstep.Step{ID: "mvl0frv18d1973", Node: "ace", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Started: time.Now()}) - if _, err := (Asker{Publisher: h, Waiter: h, Bound: 20 * time.Millisecond}).Wait(context.Background(), goAsk("mvl0frv18d1973")); err == nil || !strings.Contains(err.Error(), "given again after a transport fault") { + if _, err := (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: 20 * time.Millisecond}).Wait(context.Background(), goAsk("mvl0frv18d1973")); err == nil || !strings.Contains(err.Error(), "given again after a transport fault") { t.Errorf("an ask never given again: %v", err) } // ADR 0664 §5: that ask is dropped; a wait ended by a named cause is dropped too, an ask no node took included (it waits in @@ -119,19 +119,19 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { drops := &seatDrops{} h = &FakeAsks{} h.Write(buildstep.Step{ID: "mvl0frv18d1974", Node: "ace", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Started: time.Now()}) - (Asker{Publisher: h, Waiter: h, Bound: 20 * time.Millisecond, Dropper: drops}).Wait(context.Background(), goAsk("mvl0frv18d1974")) + (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: 20 * time.Millisecond, Dropper: drops}).Wait(context.Background(), goAsk("mvl0frv18d1974")) named, stop := context.WithCancelCause(context.Background()) h.Write(buildstep.Step{ID: "mvl0frv18d1975", Node: "g14", State: buildstep.StepRunning, Started: time.Now()}) seenStep := WithStepSeen(named, func(buildstep.Step) { stop(Withdrawn{Why: "superseded by a newer fingerprint"}) }) - (Asker{Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(seenStep, goAsk("mvl0frv18d1975")) + (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(seenStep, goAsk("mvl0frv18d1975")) shut, off := context.WithCancel(context.Background()) off() - (Asker{Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(shut, goAsk("mvl0frv18d1977")) - (Asker{Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(named, goAsk("mvl0frv18d1976")) + (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(shut, goAsk("mvl0frv18d1977")) + (Asker{Seats: testSeats(), Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(named, goAsk("mvl0frv18d1976")) // Review 1 (finding 1): a cause that is no withdrawal (the builder's check bound passing, which waits on) drops nothing. bounded, passed := context.WithCancelCause(context.Background()) passed(errors.New("the check bound passed")) - (Asker{Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(bounded, goAsk("mvl0frv18d1978")) + (Asker{Seats: testSeats(), Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(bounded, goAsk("mvl0frv18d1978")) if got := strings.Join(drops.got, "|"); !strings.Contains(got, "mvl0frv18d1974") || !strings.Contains(got, "mvl0frv18d1975 superseded by a newer fingerprint") || !strings.Contains(got, "mvl0frv18d1976 superseded by a newer fingerprint") || len(drops.got) != 3 { t.Errorf("drops %q, want the ask never given again, the superseded one and the one no node took, and no other", got) } @@ -139,20 +139,20 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // how many runs; a drop that failed is said, not discarded. late := &FakeAsks{} late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"}) - if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") { + if st, err := (Asker{Seats: testSeats(), Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") { t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err) } var said []string bad, off2 := context.WithCancelCause(context.Background()) off2(Withdrawn{Why: "withdrawn"}) - (Asker{Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: failingDrops{}, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Wait(bad, goAsk("mvl0frv18d1980")) + (Asker{Seats: testSeats(), Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: failingDrops{}, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Wait(bad, goAsk("mvl0frv18d1980")) if len(said) != 1 || !strings.Contains(said[0], "mvl0frv18d1980") || !strings.Contains(said[0], "no holder") { t.Errorf("a drop that failed said %q, want it named with its cause", said) } // review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step // running, the step is bounded from its start, and the error names the node. g := &FakeAsks{} - k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond} + k = Asker{Seats: testSeats(), Publisher: g, Waiter: g, Bound: 20 * time.Millisecond} waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded { @@ -170,7 +170,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // the phase; one three bounds from its start ends, said as kept moving. The steps' own times stand for the clock, so the // bound is an hour no loaded machine reaches and no case waits on a pause (the check of 2026-11-02 22:35 tripped one // of 60 ms). - k = Asker{Publisher: g, Waiter: g, Bound: time.Hour} + k = Asker{Seats: testSeats(), Publisher: g, Waiter: g, Bound: time.Hour} now := time.Now() g.Write(buildstep.Step{ID: "mvl0frv18d1980", Node: "ace", State: buildstep.StepRunning, Started: now.Add(-90 * time.Minute), Phase: "pb", PhaseAt: now}) g.Write(buildstep.Step{ID: "mvl0frv18d1980", Node: "ace", State: buildstep.StepPassed}) @@ -190,7 +190,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // first is answered as the node wrote it. dead := &DeadLetters{} lw := LostAskWaiter{Inner: g, Dead: dead} - k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute} + k = Asker{Seats: testSeats(), Publisher: g, Waiter: lw, Bound: time.Minute} g.Write(buildstep.Step{ID: "mvl0frv18d1970", Node: "ace", State: buildstep.StepRunning, Started: time.Now()}) given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"}, "Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}}) @@ -225,7 +225,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") { t.Fatalf("a refused hearing says %q", why) } - k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute} + k = Asker{Seats: testSeats(), Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute} g.Write(buildstep.Step{ID: "mvl0frv18d1973", Node: "g14", State: buildstep.StepPassed}) if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil { t.Fatal(err) @@ -245,25 +245,25 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again. d := &FakeAsks{} heard := &HeardEnds{Inner: deafWatch{}, Read: d} - k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute} + k = Asker{Seats: testSeats(), Publisher: d, Waiter: heard, Bound: time.Minute} d.Write(buildstep.Step{ID: "mvl0frv18d1980", Node: "shanks", State: buildstep.StepPassed}) got := make(chan buildstep.Step, 1) go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }() for heard.Waiting() == 0 { time.Sleep(time.Millisecond) } - heard.Nudge(NodeBuildModule) + heard.Nudge(testNode) select { case s := <-got: t.Fatalf("a nudge of another build module answered %+v", s) case <-time.After(20 * time.Millisecond): } - heard.Nudge(GoBuildModule) + heard.Nudge(testGo) if s := <-got; s.State != buildstep.StepPassed || s.Node != "shanks" { t.Fatalf("a step its watch never handed over: %+v", s) } closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d} - if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != buildstep.StepPassed { + if s, err := (Asker{Seats: testSeats(), Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != buildstep.StepPassed { t.Fatalf("a watch that closed once: %+v, %v", s, err) } for _, row := range []struct { @@ -287,7 +287,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // bound is reached only by a wait that kept the first run's start. func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) { f := &FakeAsks{} - k := Asker{Publisher: f, Waiter: f, Bound: time.Hour} + k := Asker{Seats: testSeats(), Publisher: f, Waiter: f, Bound: time.Hour} const id = "mvl0frv18d1962" now := time.Now() f.Write(buildstep.Step{ID: id, Node: "ace", State: buildstep.StepRunning, Started: now}) diff --git a/modules/mesh-delivery-builder/buildrun/buildseats.go b/modules/mesh-delivery-builder/buildrun/buildseats.go new file mode 100644 index 00000000..b4c75e11 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/buildseats.go @@ -0,0 +1,166 @@ +package buildrun + +import ( + "fmt" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// **The builder names no build module** (design 47 G §7, §428, §704; A-82): a build module is the holder of the build seat +// whose acting rule `build` has an `on` matching an artifact's language or `kind:image`, found on the seat list (the seat +// definitions) and the catalog half (the manifest that claims the seat), as the indexer's seed finds it. Every task's module and +// every lane's key is that holder's name as the seat list and the catalog half give it. + +// BuildModule is one build seat and the module that holds it. +type BuildModule struct { + Seat string // the seat whose acting rule builds + Module string // the module holding the seat, empty while the catalog half names none + // Toolchain is the language of the toolchain the rule's `on` names, empty for a rule on `kind:image` (no toolchain). + Toolchain string + // ByKind is whether the rule is on `kind:image`: its artifacts have no language and its nodes come from the node-engine's have. + ByKind bool +} + +// AskLanguage is the language of the seat's asks (sdkstep.AskSubject): the seat is named node-build-. +func (m BuildModule) AskLanguage() string { return strings.TrimPrefix(m.Seat, "node-build-") } + +// BuildSeats finds the build modules from the seat definitions and the holders of seats, both read at each call. +type BuildSeats struct { + // Defs are the seat definitions of the seat list; Holder is a seat's holder on the catalog half. + defs func() map[string]seatdef.Definition + holder func(seat string) (string, bool) +} + +// NewBuildSeats is the finder over defs (the seat list's definitions, nil before it is read) and holder. +func NewBuildSeats(defs func() map[string]seatdef.Definition, holder func(seat string) (string, bool)) *BuildSeats { + return &BuildSeats{defs: defs, holder: holder} +} + +func (s *BuildSeats) all() []BuildModule { + if s == nil || s.defs == nil { + return nil + } + defs := s.defs() + var out []BuildModule + for _, seat := range slices.Sorted(maps.Keys(defs)) { + for _, r := range defs[seat].Acts { + if r.Step != seatdef.StepBuild || r.Acting() == seatdef.OnEvery { + continue + } + m := BuildModule{Seat: seat, ByKind: r.On == seatdef.OnKindImage} + if !m.ByKind { + m.Toolchain = r.Acting() + } + if s.holder != nil { + m.Module, _ = s.holder(seat) + } + out = append(out, m) + break + } + } + return out +} + +// All are the build seats with the module holding each, by seat name; a seat no catalog entry claims has no module. +func (s *BuildSeats) All() []BuildModule { return s.all() } + +// Modules are the holders of the build seats, sorted, none that is unnamed. +func (s *BuildSeats) Modules() []string { + var out []string + for _, m := range s.all() { + if m.Module != "" { + out = append(out, m.Module) + } + } + slices.Sort(out) + return slices.Compact(out) +} + +// OfModule is the build seat held by module. +func (s *BuildSeats) OfModule(module string) (BuildModule, bool) { + for _, m := range s.all() { + if m.Module == module && module != "" { + return m, true + } + } + return BuildModule{}, false +} + +// OfSeat is the build seat named seat. +func (s *BuildSeats) OfSeat(seat string) (BuildModule, bool) { + for _, m := range s.all() { + if m.Seat == seat { + return m, true + } + } + return BuildModule{}, false +} + +// For is the build module of an artifact: the build seat whose rule's `on` is language, or `kind:image` when the artifact's +// kind is image. None, or more than one, or a seat no entry claims, is the error. +func (s *BuildSeats) For(language, kind string) (BuildModule, error) { + var found []BuildModule + for _, m := range s.all() { + if (kind == "image" && m.ByKind) || (!m.ByKind && language != "" && strings.EqualFold(m.Toolchain, strings.TrimSpace(language))) { + found = append(found, m) + } + } + switch len(found) { + case 0: + return BuildModule{}, fmt.Errorf("no seat builds an artifact of language %q and kind %q", language, kind) + case 1: + if found[0].Module == "" { + return BuildModule{}, fmt.Errorf("no module claims %s, which builds an artifact of language %q and kind %q", found[0].Seat, language, kind) + } + return found[0], nil + } + return BuildModule{}, fmt.Errorf("the seats %v each build an artifact of language %q and kind %q", seatsOf(found), language, kind) +} + +func seatsOf(ms []BuildModule) []string { + var out []string + for _, m := range ms { + out = append(out, m.Seat) + } + return out +} + +// LanguageOf is the language of the toolchain the build module builds with, false for a module that is no build seat's holder +// or builds with none. +func (s *BuildSeats) LanguageOf(module string) (string, bool) { + m, ok := s.OfModule(module) + return m.Toolchain, ok && m.Toolchain != "" +} + +// Languages are the languages of the toolchains the build seats' rules name, sorted. +func (s *BuildSeats) Languages() []string { + var out []string + for _, m := range s.all() { + if m.Toolchain != "" { + out = append(out, m.Toolchain) + } + } + slices.Sort(out) + return slices.Compact(out) +} + +// OfAskLanguage is the build module whose seat takes the asks of an ask language (go, node or image): the seat the sdk names +// for it (buildstep.BuildSeat), held by the module the catalog half names. +func (s *BuildSeats) OfAskLanguage(language string) (BuildModule, error) { + seat, err := buildstep.BuildSeat(language) + if err != nil { + return BuildModule{}, err + } + m, ok := s.OfSeat(seat) + if !ok { + return BuildModule{}, fmt.Errorf("the seat list holds no build seat %s", seat) + } + if m.Module == "" { + return BuildModule{}, fmt.Errorf("no module claims %s", seat) + } + return m, nil +} diff --git a/modules/mesh-delivery-builder/buildrun/buildseats_test.go b/modules/mesh-delivery-builder/buildrun/buildseats_test.go new file mode 100644 index 00000000..aa6d7a88 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/buildseats_test.go @@ -0,0 +1,60 @@ +package buildrun + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The build module of an artifact is the holder of the build seat whose rule `on` names its language, or kind:image, as the seat +// list and the catalog half name it (design 47 G A-82): no module name is in the code. +func TestTheBuildModuleIsTheHolderOfTheBuildSeat(t *testing.T) { + rule := func(step, on string) seatdef.Definition { + return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: step, On: on}}} + } + defs := map[string]seatdef.Definition{ + "node-build-go": rule(seatdef.StepBuild, "go"), "node-build-node": rule(seatdef.StepBuild, "typescript"), + "node-build-image": rule(seatdef.StepBuild, seatdef.OnKindImage), "node-build-rust": rule(seatdef.StepBuild, "go"), + "mesh-checks": rule(seatdef.StepCheck, "go"), "node-builder-every": rule(seatdef.StepBuild, "every"), + } + holders := map[string]string{"node-build-go": "acme-go", "node-build-node": "acme-ts", "node-build-image": "acme-img"} + seats := NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(s string) (string, bool) { h, ok := holders[s]; return h, ok }) + for _, c := range []struct{ language, kind, module, fail string }{ + {"typescript", "bundle", "acme-ts", ""}, + {"", "image", "acme-img", ""}, + {"python", "bundle", "", "no seat builds"}, + {"go", "bundle", "", "each build"}, // node-build-go and node-build-rust both build go + } { + got, err := seats.For(c.language, c.kind) + switch { + case c.fail != "" && (err == nil || !strings.Contains(err.Error(), c.fail)): + t.Errorf("For(%q, %q) = %+v, %v; want the error %q", c.language, c.kind, got, err, c.fail) + case c.fail == "" && (err != nil || got.Module != c.module): + t.Errorf("For(%q, %q) = %+v, %v; want %s", c.language, c.kind, got, err, c.module) + } + } + delete(defs, "node-build-rust") + if m, err := seats.For("go", "bundle"); err != nil || m.Module != "acme-go" || m.Toolchain != "go" || m.AskLanguage() != "go" { + t.Errorf("For(go) = %+v, %v", m, err) + } + if l, ok := seats.LanguageOf("acme-ts"); !ok || l != "typescript" { + t.Errorf("LanguageOf(acme-ts) = %q, %v", l, ok) + } + if _, ok := seats.LanguageOf("acme-img"); ok { + t.Error("an image build module has a toolchain language") + } + if got := strings.Join(seats.Modules(), ","); got != "acme-go,acme-img,acme-ts" { + t.Errorf("Modules = %s", got) + } + delete(holders, "node-build-go") // no catalog entry claims the seat: no build module, said by name + if _, err := seats.For("go", "bundle"); err == nil || !strings.Contains(err.Error(), "no module claims node-build-go") { + t.Errorf("a seat with no holder: %v", err) + } + if m, err := seats.OfAskLanguage("image"); err != nil || m.Module != "acme-img" { + t.Errorf("OfAskLanguage(image) = %+v, %v", m, err) + } + if got := NewBuildSeats(func() map[string]seatdef.Definition { return nil }, nil).Modules(); len(got) != 0 { + t.Errorf("before the seat list is read: %v", got) + } +} diff --git a/modules/mesh-delivery-builder/buildrun/heardends.go b/modules/mesh-delivery-builder/buildrun/heardends.go index bdca8068..f9c16ea6 100644 --- a/modules/mesh-delivery-builder/buildrun/heardends.go +++ b/modules/mesh-delivery-builder/buildrun/heardends.go @@ -36,6 +36,9 @@ type HeardEnds struct { mu sync.Mutex waiting map[string]*waitedAsk // by ask id + watched map[string]bool // the modules whose steps are watched + asks JetStreamAsks + ctx context.Context } type waitedAsk struct { @@ -180,16 +183,31 @@ func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules if logf == nil { logf = func(string, ...any) {} } - h := &HeardEnds{Inner: inner, Read: asks, Logf: logf} + h := &HeardEnds{Inner: inner, Read: asks, Logf: logf, asks: asks, ctx: ctx, watched: map[string]bool{}} + h.Watch(modules) + return h +} + +// Watch starts the watch of the steps of each module not watched yet: the build modules are the holders of the build seats the +// seat list and the catalog half name (design 47 G A-82), known only once both are read, and again at each change of them. +func (h *HeardEnds) Watch(modules []string) { for _, module := range modules { - kv, err := asks.JS.KeyValue(ctx, StepsBucket(module)) - if err != nil { - logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err) + h.mu.Lock() + done := h.watched[module] + h.mu.Unlock() + if done { continue } - go h.follow(ctx, kv, module) + kv, err := h.asks.JS.KeyValue(h.ctx, StepsBucket(module)) + if err != nil { + h.logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err) + continue + } + h.mu.Lock() + h.watched[module] = true + h.mu.Unlock() + go h.follow(h.ctx, kv, module) } - return h } func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) { diff --git a/modules/mesh-delivery-builder/buildrun/modules.go b/modules/mesh-delivery-builder/buildrun/modules.go deleted file mode 100644 index db15aedb..00000000 --- a/modules/mesh-delivery-builder/buildrun/modules.go +++ /dev/null @@ -1,7 +0,0 @@ -package buildrun - -// The build modules the mesh runs, by the language each builds (ADR 0469). -const ( - GoBuildModule = "mesh-build-go" - NodeBuildModule = "mesh-build-node" -) diff --git a/modules/mesh-delivery-builder/buildrun/seats_test.go b/modules/mesh-delivery-builder/buildrun/seats_test.go new file mode 100644 index 00000000..37096b8c --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/seats_test.go @@ -0,0 +1,22 @@ +package buildrun + +import ( + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82). +const ( + testGo = "mesh-build-go" + testNode = "mesh-build-node" + testImage = "mesh-build-image" +) + +// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module. +func testSeats() *BuildSeats { + rule := func(on string) seatdef.Definition { + return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}} + } + defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)} + holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage} + return NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }) +} diff --git a/modules/mesh-delivery-builder/buildrun/standing_test.go b/modules/mesh-delivery-builder/buildrun/standing_test.go index 2b8da16f..f2ceee5d 100644 --- a/modules/mesh-delivery-builder/buildrun/standing_test.go +++ b/modules/mesh-delivery-builder/buildrun/standing_test.go @@ -41,7 +41,7 @@ func TestPlacedNodes(t *testing.T) { t.Run("the assignments names the counted nodes once it is settled", func(t *testing.T) { gate := NewPlacedNodes(func(string, ...any) {}) for _, node := range three { - k, v := placing(t, node, NodeBuildModule) + k, v := placing(t, node, testNode) if gate.Assignments(k, v, false, old) { t.Fatal("an assignments before it is settled changed the counted nodes") } @@ -50,7 +50,7 @@ func TestPlacedNodes(t *testing.T) { if got := strings.Join(gate.Nodes(), ","); got != "g14,novox,shanks" { t.Fatalf("nodes = %q", got) } - k, v := placing(t, "ace", GoBuildModule) + k, v := placing(t, "ace", testGo) if !gate.Assignments(k, v, false, old) || len(gate.Nodes()) != 4 { t.Fatalf("ace placed: nodes %v", gate.Nodes()) } @@ -65,7 +65,7 @@ func TestPlacedNodes(t *testing.T) { at := old.Add(time.Duration(n) * time.Hour) r := meshwants.AssignmentRevision{Revision: n, Kind: "trunk", At: &at} for _, node := range nodes { - r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: NodeBuildModule, Digest: "d-" + node}) + r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: testNode, Digest: "d-" + node}) } return r } diff --git a/modules/mesh-delivery-builder/buildrun/toolchaingate_test.go b/modules/mesh-delivery-builder/buildrun/toolchaingate_test.go index 9e6521c3..90f6a73a 100644 --- a/modules/mesh-delivery-builder/buildrun/toolchaingate_test.go +++ b/modules/mesh-delivery-builder/buildrun/toolchaingate_test.go @@ -42,16 +42,16 @@ func TestToolchainGate(t *testing.T) { placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again three := []string{"g14", "novox", "shanks"} start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) { - set := &ToolchainSet{} + set := &ToolchainSet{Seats: testSeats()} said := &[]string{} - gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) + gate := NewToolchainGate([]string{testGo, testNode}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) for _, node := range three { if seeded { - key, value := placing(t, node, NodeBuildModule) + key, value := placing(t, node, testNode) gate.Assignments(key, value, false, old.Add(-time.Hour)) } key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute)) - gate.Toolchain(NodeBuildModule, key, value, false) + gate.Toolchain(testNode, key, value, false) } return gate, set, said } @@ -59,11 +59,11 @@ func TestToolchainGate(t *testing.T) { t.Run("not placed on ace: its report is no builder's", func(t *testing.T) { gate, set, said := start(t, true) key, value := gateReport(t, "ace", "v26.12.0", old) - gate.Toolchain(NodeBuildModule, key, value, false) + gate.Toolchain(testNode, key, value, false) if a := buildstep.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out t.Fatalf("before the assignments is settled every report counts; got %+v", a) } - k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter + k, v := placing(t, "ace", testGo) // another build module on ace: no matter gate.Assignments(k, v, false, old) if !gate.Settled() { t.Fatal("settling left ace's report out, so what was passed on changed") @@ -74,11 +74,11 @@ func TestToolchainGate(t *testing.T) { if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") { t.Fatalf("ace's report is said left out once; said %q", *said) } - if gate.Toolchain(NodeBuildModule, key, value, false) { + if gate.Toolchain(testNode, key, value, false) { t.Fatal("a report of a node not placed changed what was passed on") } // Placed on ace with a report written since: it counts, and the builders really disagree. - k, v = placing(t, "ace", GoBuildModule, NodeBuildModule) + k, v = placing(t, "ace", testGo, testNode) gate.Assignments(k, v, false, old.Add(-time.Minute)) if a := buildstep.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" { t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a) @@ -92,24 +92,24 @@ func TestToolchainGate(t *testing.T) { } // A report deleted from the bucket goes whether its node is placed or not. key, _ = gateReport(t, "g14", "v26.12.1", placed) - if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 { + if !gate.Toolchain(testNode, key, nil, true) || len(set.Entries("typescript")) != 2 { t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript"))) } }) t.Run("build modules given later pass their reports on when the assignments moves", func(t *testing.T) { - set := &ToolchainSet{} + set := &ToolchainSet{Seats: testSeats()} gate := NewToolchainGate(nil, set.Put, func(string, ...any) {}) gate.Settled() - k, v := placing(t, "novox", GoBuildModule) + k, v := placing(t, "novox", testGo) gate.Assignments(k, v, false, old) key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) - gate.Toolchain(NodeBuildModule, key, value, false) // ace is not placed: left out - k, v = placing(t, "ace", NodeBuildModule) + gate.Toolchain(testNode, key, value, false) // ace is not placed: left out + k, v = placing(t, "ace", testNode) if gate.Assignments(k, v, false, placed) || len(set.Entries("typescript")) != 0 { t.Fatalf("a module the gate was not given passed on at the assignments: %v", set.Entries("typescript")) } - if !gate.UseModules([]string{NodeBuildModule}) || len(set.Entries("typescript")) != 1 { + if !gate.UseModules([]string{testNode}) || len(set.Entries("typescript")) != 1 { t.Fatalf("given mesh-build-node, ace's report placed since is passed on; entries %v", set.Entries("typescript")) } }) @@ -127,16 +127,16 @@ func TestToolchainGate(t *testing.T) { k, v := placing(t, "ace") // ace's key, first heard placing nothing gate.Assignments(k, v, false, old.Add(-time.Hour)) gate.Settled() - k, v = placing(t, "ace", NodeBuildModule) + k, v = placing(t, "ace", testNode) if gate.Assignments(k, v, false, placed) { t.Fatal("assignments with no report passed something on") } key, value := gateReport(t, "ace", "v26.12.0", c.at) - if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts { + if got := gate.Toolchain(testNode, key, value, false); got != c.counts { t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) } // ace's assignments written again with another module beside: its standing does not move, nothing is said again. - k, v = placing(t, "ace", GoBuildModule, NodeBuildModule) + k, v = placing(t, "ace", testGo, testNode) if gate.Assignments(k, v, false, placed.Add(time.Minute)) { t.Fatal("the same assignments of mesh-build-node changed what was passed on") } @@ -150,7 +150,7 @@ func TestToolchainGate(t *testing.T) { t.Fatalf("said %q, want once why ace's report is left out", *said) } key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) - if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 { + if !gate.Toolchain(testNode, key, value, false) || len(set.Entries("typescript")) != 4 { t.Fatal("ace's new report was not passed on") } }) @@ -159,12 +159,12 @@ func TestToolchainGate(t *testing.T) { t.Run("no assignments held: every report counts", func(t *testing.T) { gate, set, _ := start(t, false) key, value := gateReport(t, "ace", "v26.12.0", old) - gate.Toolchain(NodeBuildModule, key, value, false) + gate.Toolchain(testNode, key, value, false) gate.Settled() if n := len(set.Entries("typescript")); n != 4 { t.Fatalf("before the first assignments file every report counts; got %d", n) } - k, v := placing(t, "g14", NodeBuildModule) + k, v := placing(t, "g14", testNode) if !gate.Assignments(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 { t.Fatalf("the first assignments leaves every node it does not place out; got %d", len(set.Entries("typescript"))) } @@ -177,20 +177,20 @@ func TestToolchainGate(t *testing.T) { // leaves out a node no revision places the build module on, and counts a node placed in a later revision only by a // report written since that revision's time. t.Run("fed by the assignment half", func(t *testing.T) { - set := &ToolchainSet{} + set := &ToolchainSet{Seats: testSeats()} said := &[]string{} - gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) + gate := NewToolchainGate([]string{testGo, testNode}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) for _, node := range append([]string{"ace"}, three...) { key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute)) if node == "ace" { key, value = gateReport(t, node, "v26.12.0", old) } - gate.Toolchain(NodeBuildModule, key, value, false) + gate.Toolchain(testNode, key, value, false) } revision := func(n int64, at time.Time, nodes ...string) meshwants.AssignmentRevision { r := meshwants.AssignmentRevision{Revision: n, Kind: "trunk", At: &at} for _, node := range nodes { - r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: NodeBuildModule, Digest: "d-" + node}) + r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: testNode, Digest: "d-" + node}) } return r } @@ -207,7 +207,7 @@ func TestToolchainGate(t *testing.T) { t.Fatalf("said %q, want why ace's old report is left out", *said) } key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) - if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 { + if !gate.Toolchain(testNode, key, value, false) || len(set.Entries("typescript")) != 4 { t.Fatal("ace's report written since revision 8 is not passed on") } if !gate.Revision(revision(9, placed.Add(time.Hour), "ace", "novox", "shanks")) || len(set.Entries("typescript")) != 3 || len(gate.Nodes()) != 3 { @@ -218,27 +218,27 @@ func TestToolchainGate(t *testing.T) { t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) { p := meshmoduleassignerclient.NewPlaced() // ace's node key is the first revision heard of it: its module stands since before. - k, v := placing(t, "ace", NodeBuildModule) + k, v := placing(t, "ace", testNode) p.Put(k, v, false, old) - if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 { + if _, changed, _ := p.Put("ace."+testNode, []byte("{}"), false, placed); len(changed) != 0 { t.Errorf("a module key says which modules stand: %v", changed) } k2, v2 := placing(t, "g14") p.Put(k2, v2, false, old) - k2, v2 = placing(t, "g14", NodeBuildModule) - if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule { + k2, v2 = placing(t, "g14", testNode) + if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != testNode { t.Errorf("a module placed says changed %v", changed) } if _, _, err := p.Put("novox.node", []byte("not json"), false, old); err == nil { t.Error("a value that does not read was not said") } - if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() { + if s, ok := p.Since("ace", testNode); !ok || !s.IsZero() { t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok) } - if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) { + if s, ok := p.Since("g14", testNode); !ok || !s.Equal(placed) { t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok) } - if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() { + if got := p.PlacedNodes(testNode); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() { t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode()) } }) diff --git a/modules/mesh-delivery-builder/buildrun/toolchains.go b/modules/mesh-delivery-builder/buildrun/toolchains.go index af1f53e6..12b47bb5 100644 --- a/modules/mesh-delivery-builder/buildrun/toolchains.go +++ b/modules/mesh-delivery-builder/buildrun/toolchains.go @@ -7,9 +7,6 @@ import ( "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) -// LanguageOf is the language each build module's toolchain reports hold. -var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"} - // ToolchainsHeardState is the builder's old state section of the toolchains it heard (design 47 G §604, §546): one key // ., the toolchain as the build seat said it. const ( @@ -20,8 +17,10 @@ const ( // ToolchainSet holds each node's toolchain of a language, as the builder heard it (design 47 G §604). Safe for use from // the consumer and from a build at once. type ToolchainSet struct { - mu sync.Mutex - by map[string]map[string]buildstep.Toolchain // language -> node -> entry + // Seats finds the build module of a language: a report of a module that holds no build seat is left out (design 47 G A-82). + Seats *BuildSeats + mu sync.Mutex + by map[string]map[string]buildstep.Toolchain // language -> node -> entry } // Set takes a node's toolchain in and answers whether the language's entries changed (a node's version, recipe or @@ -96,7 +95,7 @@ const ( // delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going). // A key or a value that is not an entry of the module's language is left out. func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool { - language, ok := LanguageOf[module] + language, ok := s.Seats.LanguageOf(module) if !ok { return false } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 29740838..23cf771f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -22,6 +22,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" @@ -336,12 +338,8 @@ const ( ) func main() { - steps := map[string]build.StateRead{} - for _, bm := range build.StuckBuildModules { - steps[bm] = read(stdio.State(bm + ".steps")) - } reasks := stdio.State(build.ReAsksState) - r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps, + r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), StepsOf: func(seat string) build.StateRead { return read(stdio.State(seat + ".steps")) }, Hearing: read(stdio.State(buildrun.HearingState))} put := func(key string, value []byte) error { _, err := reasks.Put(key, json.RawMessage(value)) @@ -400,6 +398,19 @@ func main() { os.Exit(1) } pk.Start(context.Background()) + // The build modules are no names (design 47 G A-82): the build seats on the seat list, each held by the module the catalog + // half's manifests say; the seat list and its two tools wired once by mesh-sdk (§204). + gate := seatlist.NewPart("builder", []string{stdio.AnyPerson, "node-tools", "mesh-healers"}) + go func() { + <-stdio.Served() + gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }) + }() + r.Seats = buildrun.NewBuildSeats(func() map[string]seatdef.Definition { + if rd := gate.Reader(); rd != nil { + return rd.Definitions() + } + return nil + }, versions.HolderOf(branches)) carriers := versions.CarriersFrom(versions.EntriesOf(branches)) released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) { raw, err := stdio.Ask(catalogVersionsVerb, meshcatalog.VersionsInput{Module: module}) @@ -445,6 +456,7 @@ func main() { // builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's // states stand, and asking one again by hand (ADR 0615, DLV-M1). tools = append(tools, views.StatusTools(builds.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + tools = append(tools, gate.Tools()...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go index cf2693f4..f7e12494 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -74,6 +74,8 @@ type catalogHalf struct { halves map[string]*views.Follower once sync.Once ctx context.Context + // holders is told each entry change: a claim of a build seat may have moved (design 47 G A-82). + holders func() } // first says whether branch is the first watched branch of module's registration (design 47 G §371); a module with no @@ -86,6 +88,9 @@ func (c *catalogHalf) first(branch, module string) bool { // Changed is the follower's Changed: one module's entry of a branch came, or went. func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, entry json.RawMessage, deleted bool, at time.Time) { return func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) { + if c.holders != nil { + defer c.holders() + } first := c.first(branch, module) if deleted { c.b.EntryGone(branch, first, module) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go index 6d2950fe..9d74cd2e 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go @@ -5,6 +5,8 @@ package main import ( "context" "encoding/json" + "fmt" + "strings" "sync" "testing" "time" @@ -14,6 +16,9 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest" ) // catalogHolder is the holder of the seat mesh-catalog as the bed needs it (design 47 G §348, §614): it answers `revision` for the @@ -92,3 +97,48 @@ func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { t.Fatal(err) } } + +// buildSeats is the seat list and the catalog half as the builder finds its build modules from them (design 47 G A-82): the seat +// mesh-catalog's verb seat-definitions answers the three build seats' definitions, each acting on its language (the image seat on +// kind:image), and an entry of each holder's manifest claims its seat. +func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[string]string) { + t.Helper() + defs := map[string]seatdef.Definition{} + for _, d := range seatlisttest.Definitions() { + if _, ok := holders[d.Seat]; ok { + defs[d.Seat] = d + } + } + for seat, on := range map[string]string{"node-build-go": "go", "node-build-node": "typescript", "node-build-image": seatdef.OnKindImage} { + d := defs[seat] + d.Acts = []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}} + defs[seat] = d + } + s, err := nc.Subscribe("mesh.seat."+meshcatalog.Seat+".call."+seatlist.Verb+".>", func(m *nats.Msg) { + var in struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(m.Data, &in) + var result any + if in.Seat == "" { + l := seatlist.Listing{Revision: 1} + for seat := range defs { + l.Seats = append(l.Seats, seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "sha256:" + seat}) + } + result = l + } else { + raw, _ := json.Marshal(defs[in.Seat]) + result = seatlist.Answer{Seat: in.Seat, Found: true, Revision: 1, Entry: &seatlist.Entry{Seat: in.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:" + in.Seat, Definition: raw}} + } + reply, _ := json.Marshal(map[string]any{"result": result, "node": "ctl"}) + _ = m.Respond(reply) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Unsubscribe() }) + for seat, module := range holders { + h.put(t, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", len(seat)%16), 64), ManifestDigest: "sha256:claims-" + module, + Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}}) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go index f8cb1e46..475c7050 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -19,7 +19,10 @@ import ( // heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it // (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still // be answered (issue 1945). Never polled. -type heldOnTheBus struct{ js jetstream.JetStream } +type heldOnTheBus struct { + js jetstream.JetStream + seats *buildrun.BuildSeats // the build modules (design 47 G A-82) +} // Held is build.Builder's Held: a step key `.` in the build module's steps, or a message of the stream under // the ask's subject whose Nats-Msg-Id is the ask's id. @@ -51,10 +54,11 @@ const ( // the ask is on its stream. func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool) (build.AskSeen, error) { seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)} - module, err := buildstep.BuildModuleOf(a.Language) + bm, err := h.seats.OfAskLanguage(a.Language) if err != nil { return seen, err } + module := bm.Module steps, err := h.js.KeyValue(ctx, buildrun.StepsBucket(module)) if err != nil { return seen, fmt.Errorf("the bucket %s: %w", buildrun.StepsBucket(module), err) @@ -259,10 +263,11 @@ func (h heldOnTheBus) Drop(ctx context.Context, a buildstep.StepAsk) error { // every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits. func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) { q := build.SeatQueue{Language: language, Note: build.QueueNote} - module, err := buildstep.BuildModuleOf(language) + bm, err := h.seats.OfAskLanguage(language) if err != nil { return q, err } + module := bm.Module stream, name, _, err := h.stream(ctx, language, buildstep.AskDelivery) q.Stream = name if errors.Is(err, jetstream.ErrStreamNotFound) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 3d5831a7..d53136cf 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -303,6 +303,8 @@ func startBedWith(t *testing.T, closed bool) *bed { // verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2 // each have their durable consumer. b.catalog = newCatalogHolder(t, ctx, js, ctl) + b.catalog.buildSeats(t, ctl, map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage, + "git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}) // the floor's sources are the holders of git and mesh-catalog readers := []string{"n1_" + builds.BuilderModule, "n2_" + builds.BuilderModule} viewstest.Serve(t, ctx, js, ctl, assignerModule, "assignments", assignerModule+".assigner_read_assignments", readers...) b.builds = mk(builds.BuilderBuildsBucket, builds.BuilderBuildsHistory) @@ -1001,7 +1003,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { } // 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can // still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream. - held := heldOnTheBus{b.js} + held := heldOnTheBus{js: b.js, seats: testSeats()} goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, sdkstep.AskerBuilder) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 9a5c7818..d471a5d4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -38,6 +38,7 @@ import ( "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-builder/internal/build" @@ -115,6 +116,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } branches.Logf = logger.Printf half.branches = branches + // The build modules are no names (design 47 G A-82): the holders of the build seats, the seat list's definitions and the + // catalog half's claims, found at each use. + seatGate := newSeatGate() + seats := buildrun.NewBuildSeats(seatGate.definitions, versions.HolderOf(branches)) assignments, err := reader.Follow(assignerModule, "assignments", assignerModule+".assigner_read_assignments") if err != nil { return err @@ -125,10 +130,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err) } } - tools := &buildrun.ToolchainSet{} + tools := &buildrun.ToolchainSet{Seats: seats} // Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot // (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity. asks := buildrun.JetStreamAsks{JS: js} + // The steps of each build module are watched once the seat list and the catalog half name it (seatsChanged). // The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once, // and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound. // Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the @@ -142,13 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer hearing.Stop() // Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read // (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28). - var buildModules []string - for _, l := range buildstep.AskLanguages { - if m, err := buildstep.BuildModuleOf(l); err == nil { - buildModules = append(buildModules, m) - } - } - heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf) + heard := buildrun.HearEnds(ctx, asks, asks, seats.Modules(), logger.Printf) conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") }) // The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs // (the seat verb held, novox/hq issue 1937) before it leaves the key done, on its calls connection. @@ -156,7 +156,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the bus holds it; the floor's three sources asked as its own account. versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf) found := &findings{w: fw, logf: logger.Printf, now: time.Now} - floorWaits := &versions.FloorWaits{Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))} + floorWaits := &versions.FloorWaits{Holder: versions.HolderOf(branches), Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))} // The names waiting on their floor are kept in the state floor-waits (DLV-B4): a restarted builder takes them in, so // their silent sources' events still wake them; until the state is bound they are kept here only. process.AwaitBucket(ctx, js, conn, versions.FloorWaitsBucket, "the names waiting on their floor are kept in memory only, and builder_floor_waits shows none", logger.Printf). @@ -164,19 +164,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf)) }) // A package is numbered under its name in its ecosystem, read from the catalog half's entries (design 32, design 47 G §348). - numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, + numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call, versions.HolderOf(branches)), Waits: floorWaits, Names: versions.NamesFrom(versions.EntriesOf(branches))} // A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder // answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read. pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf) pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf} numberer.Publishing = pubs.arm - b := &build.Builder{ + var b *build.Builder + seatsChanged := func(ctx context.Context) { + heard.Watch(seats.Modules()) + if b != nil { + b.Reconsider(ctx) + } + } + b = &build.Builder{ Claimer: build.Claimer{Store: kvStore{buildsKV}, By: cred.Node + "." + cred.Module}, - Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls}}, - Held: heldOnTheBus{js}, + Seats: seats, + Asks: buildrun.Asker{Seats: seats, Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls}}, + Held: heldOnTheBus{js: js, seats: seats}, // An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound. - Gone: heldOnTheBus{js}, + Gone: heldOnTheBus{js: js, seats: seats}, CheckAfter: bound, Toolchains: tools, Manifests: half.manifests, @@ -266,7 +274,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); { case kv == nil: default: - says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf) + says := newSayer(ctx, kv, cred.Node, seats.Modules, logger.Printf) b.Answered = says.Answered defer func() { stopping, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) @@ -369,9 +377,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // condition keeper's condition-cleared. floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf} // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. - seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} + seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} events := func(m jetstream.Msg) { - if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) + if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } heardGone(ctx, b, m, logger.Printf) @@ -380,6 +388,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the // catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has // one taker, so no pattern here covers them). + // The seat list over the connection and heard on the hub: a change of it, as of the catalog half's holders, names the build + // modules again (design 47 G A-82, §204). + seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf) + half.holders = func() { seatsChanged(ctx) } for _, pattern := range seatEventPatterns { if err := hub.Handle(pattern, events); err != nil { return err @@ -451,7 +463,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the // builder alone may read the stream, so it answers there. process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { - eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) } + eachRead := func(e jetstream.KeyValueEntry) { + queueRead(ctx, heldOnTheBus{js: js, seats: seats}, kv, e, logger.Printf) + } if qw, err := settle(ctx, kv, eachRead); err != nil { process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again") } else { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go index 3b3efee3..00b38e80 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go @@ -27,7 +27,7 @@ func TestTheNamesAreTheManifests(t *testing.T) { } for _, read := range []string{ gateModule + ".source", - "mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", + "node-build-go.steps", "node-build-image.steps", "node-build-node.steps", } { if !slices.Contains(m.Reads, read) { t.Errorf("the builder reads %s, its manifest's reads are %v", read, m.Reads) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index af0276d6..dee20974 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -258,7 +258,8 @@ func (m *wakeMsg) Headers() nats.Header { func aFloorsWakeIsHeard(t *testing.T) { var logs bytes.Buffer - waits := &versions.FloorWaits{} + holders := map[string]string{"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"} + waits := &versions.FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }} waits.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceRequirements}}, nil, time.Unix(0, 0)) waits.Wait("m/img", versions.SeatImages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0)) f := &floorWake{b: &build.Builder{}, waits: waits, logf: log.New(&logs, "", 0).Printf} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go index 2454b21f..242cacbe 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go @@ -31,7 +31,7 @@ type sayer struct { ctx context.Context store sayStore node string - need []string + need func() []string logf func(string, ...any) mu sync.Mutex @@ -39,7 +39,7 @@ type sayer struct { said bool } -func newSayer(ctx context.Context, store sayStore, node string, need []string, logf func(string, ...any)) *sayer { +func newSayer(ctx context.Context, store sayStore, node string, need func() []string, logf func(string, ...any)) *sayer { return &sayer{ctx: ctx, store: store, node: node, need: need, logf: logf, answered: map[string]bool{}} } @@ -51,7 +51,7 @@ func (s *sayer) Answered(module string) { if s.said { return } - for _, m := range s.need { + for _, m := range s.need() { if !s.answered[m] { return } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says_test.go index f2199599..ce0f9f90 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says_test.go @@ -24,7 +24,7 @@ func (s saysOf) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt // one; stopped, it says nothing any more. func TestTheBuilderSaysItWritesReferencesOnceEveryBuildModuleAnsweredOne(t *testing.T) { s := saysOf{} - say := newSayer(t.Context(), s, "shanks", []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}, t.Logf) + say := newSayer(t.Context(), s, "shanks", func() []string { return []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"} }, t.Logf) say.Answered("mesh-build-go") say.Answered("mesh-build-node") if v, ok := s["shanks"]; ok { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seatlist.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seatlist.go new file mode 100644 index 00000000..7d96edc9 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seatlist.go @@ -0,0 +1,26 @@ +package main + +import ( + "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The builder reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its +// definitions name the build seats (the seats whose acting rule is the step build), its tools `builder_seat_list` and +// `builder_seat_list_again`. A call through the mesh MCP server is proved only as its machine's tool runner (issue 627). +var seatListAgainCallers = []string{stdio.AnyPerson, "node-tools", "mesh-healers"} + +// seatGate is the builder's wiring of the seat list. +type seatGate struct{ *seatlist.Part } + +func newSeatGate() *seatGate { return &seatGate{seatlist.NewPart("builder", seatListAgainCallers)} } + +// definitions are the seat definitions the list holds, none before it is read. +func (g *seatGate) definitions() map[string]seatdef.Definition { + if r := g.Reader(); r != nil { + return r.Definitions() + } + return nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats_test.go new file mode 100644 index 00000000..8bd4edf3 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82). +const ( + testGo = "mesh-build-go" + testNode = "mesh-build-node" + testImage = "mesh-build-image" +) + +// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module. +func testSeats() *buildrun.BuildSeats { + rule := func(on string) seatdef.Definition { + return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}} + } + defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)} + holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage} + return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 62c88a80..826b8392 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -104,7 +104,7 @@ func (h *toolchainsHeard) bind(ctx context.Context, state heardState) { h.mu.Lock() h.kept = state h.mu.Unlock() - for _, language := range buildrun.LanguageOf { + for _, language := range h.set.Seats.Languages() { for _, e := range h.set.Entries(language) { h.keep(e.Node, e.Toolchain) } @@ -192,7 +192,7 @@ func (h *toolchainsHeard) running(ctx context.Context, data []byte) { h.runAt[id] = r.At kept := h.kept h.mu.Unlock() - language, build := buildrun.LanguageOf[r.Module] + language, build := h.set.Seats.LanguageOf(r.Module) if !build || r.Fingerprint != "" { return } @@ -224,7 +224,7 @@ func (h *toolchainsHeard) Dropped(ctx context.Context, key string) { // agree says toolchain-agreed for each language whose agreed version is not the one last said (design 47 G §472: said // only when the agreed version changes, ADR 0584 §1). func (h *toolchainsHeard) agree(ctx context.Context) { - for _, language := range buildrun.LanguageOf { + for _, language := range h.set.Seats.Languages() { entries := h.set.Entries(language) tc, none := buildstep.AgreedToolchain(entries) if none != "" { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go index 8018670c..60cc8989 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go @@ -75,7 +75,7 @@ type heardRig struct { func newHeardRig(t *testing.T, state *memState) *heardRig { t.Helper() r := &heardRig{state: state} - r.h = newToolchainsHeard(&buildrun.ToolchainSet{}, func(context.Context) { r.changed++ }, + r.h = newToolchainsHeard(&buildrun.ToolchainSet{Seats: testSeats()}, func(context.Context) { r.changed++ }, func(_ context.Context, e meshbuilds.ToolchainAgreed) error { r.said = append(r.said, e); return nil }, log.New(io.Discard, "", 0)) r.h.bind(context.Background(), state) return r @@ -130,18 +130,18 @@ func TestTheBuilderKeepsTheToolchainsItHeardAndForgetsThemOnStopped(t *testing.T t.Fatalf("a changed agreed version was not said: %+v", again.said) } at := time.Now() - again.running(t, "ace", buildrun.GoBuildModule, "src2:x", at) // running: nothing is forgotten + again.running(t, "ace", testGo, "src2:x", at) // running: nothing is forgotten if k, _ := state.Keys(); len(k) != 3 { t.Fatalf("a running forgot a key: %v", k) } - again.running(t, "ace", buildrun.GoBuildModule, "", at.Add(time.Second)) // stopped: no fingerprint + again.running(t, "ace", testGo, "", at.Add(time.Second)) // stopped: no fingerprint if k, _ := state.Keys(); len(k) != 2 || k[0] != "g14.go" { t.Fatalf("stopped on ace left %v, want g14.go and novox.go", k) } // An older running read after (the start's read of the stream kept last per subject) changes nothing. before := again.changed - again.running(t, "g14", buildrun.GoBuildModule, "", at) - again.running(t, "g14", buildrun.GoBuildModule, "src2:y", at.Add(-time.Hour)) + again.running(t, "g14", testGo, "", at) + again.running(t, "g14", testGo, "src2:y", at.Add(-time.Hour)) if k, _ := state.Keys(); len(k) != 1 || k[0] != "novox.go" || again.changed != before+1 { t.Fatalf("after g14 stopped and an older running: %v", k) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index 24565e05..403173f7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -22,8 +22,8 @@ import ( // The floor's two module tools (novox/hq design 51 *The floor*); the seat's is `seat:.versions`. const ( - bridgeTagsTool = "mesh-delivery-forge-bridge.bridge_tags" - requirementsTool = "mesh-delivery-indexer.indexer_requirements" + bridgeTagsTool = "bridge_tags" + requirementsTool = "indexer_requirements" ) // awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue @@ -106,7 +106,15 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e // floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb // versions, the forge bridge's bridge_tags and the indexer's indexer_requirements. A call that fails is that source // silent. -func floorVia(call toolCaller) versions.FloorSources { +func floorVia(call toolCaller, holder func(seat string) (string, bool)) versions.FloorSources { + sourceTool := func(source, tool string) (string, error) { + seat := versions.SourceSeats[source] + module, ok := holder(seat) + if !ok { + return "", fmt.Errorf("no module is known to hold the seat %s, the source of %s", seat, tool) + } + return module + "." + tool, nil + } return versions.FloorSources{ Seat: func(_ context.Context, seat, name string) ([]string, error) { raw, err := call("seat:"+seat+".versions", map[string]any{"name": name}) @@ -130,7 +138,11 @@ func floorVia(call toolCaller) versions.FloorSources { return out, nil }, Tags: func(_ context.Context, repository, prefix string) ([]string, error) { - raw, err := call(bridgeTagsTool, map[string]any{"repository": repository, "prefix": prefix}) + tool, err := sourceTool(versions.SourceTags, bridgeTagsTool) + if err != nil { + return nil, err + } + raw, err := call(tool, map[string]any{"repository": repository, "prefix": prefix}) if err != nil { return nil, err } @@ -143,7 +155,11 @@ func floorVia(call toolCaller) versions.FloorSources { return a.Tags, nil }, Requirements: func(_ context.Context, name string) ([]string, error) { - raw, err := call(requirementsTool, map[string]any{"package": name}) + tool, err := sourceTool(versions.SourceRequirements, requirementsTool) + if err != nil { + return nil, err + } + raw, err := call(tool, map[string]any{"package": name}) if err != nil { return nil, err } diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index 2cf7d3e4..cddbda4d 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -10,7 +10,6 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) @@ -131,7 +130,7 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]bui if !plan.NeedsToolchain(bm) { continue } - language := buildrun.LanguageOf[bm] + language, _ := b.Seats.LanguageOf(bm) if tc, none := buildstep.AgreedToolchain(b.Toolchains.Entries(language)); none == "" { out[language] = builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} } diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index ea041b46..d8abc836 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -57,6 +57,8 @@ type ClaimOptions struct { // ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken // again only then. Nil means none does. ImageBuilder func() bool + // Seats finds the build modules of the artifacts a done key holds unpublished (design 47 G A-82). + Seats *buildrun.BuildSeats // Trunk is whether the trunk's record names this fingerprint. A done key whose image or package has no reference was // built as a candidate's, which never pushes or publishes (ADR 0487 clause 12, ADR 0498 clauses 5 and 6); on the trunk it // is taken again to publish them, an image only when an image builder runs (ImageBuilder). @@ -93,11 +95,13 @@ func GoneStep(gone []string) string { // Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has // no reference (a candidate's, ADR 0487 clause 12, ADR 0498 clauses 5 and 6). -func Unpublished(b builds.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 } +func Unpublished(b builds.BuilderBuild, seats *buildrun.BuildSeats) bool { + return len(UnpublishedModules(b, seats)) > 0 +} // UnpublishedModules are the build modules that publish what a done build holds unpublished: the image build module for // an image, the Node build module for a package; sorted, none when it holds nothing unpublished. -func UnpublishedModules(b builds.BuilderBuild) []string { +func UnpublishedModules(b builds.BuilderBuild, seats *buildrun.BuildSeats) []string { if b.State != builds.BuildDone { return nil } @@ -108,10 +112,14 @@ func UnpublishedModules(b builds.BuilderBuild) []string { } var out []string if image { - out = append(out, ImageBuildModule) + if bm, err := seats.For("", "image"); err == nil { + out = append(out, bm.Module) + } } if pkg { - out = append(out, buildrun.NodeBuildModule) + if bm, err := seats.For("typescript", catalogue.ArtifactPackage); err == nil { + out = append(out, bm.Module) + } } sort.Strings(out) return out @@ -119,10 +127,10 @@ func UnpublishedModules(b builds.BuilderBuild) []string { // publishable are the build modules of UnpublishedModules that can publish now: the image build module only while a node // runs it (imageBuilder). -func publishable(b builds.BuilderBuild, imageBuilder func() bool) []string { +func publishable(b builds.BuilderBuild, seats *buildrun.BuildSeats, imageBuilder func() bool) []string { var out []string - for _, bm := range UnpublishedModules(b) { - if bm != ImageBuildModule || (imageBuilder != nil && imageBuilder()) { + for _, bm := range UnpublishedModules(b, seats) { + if seat, _ := seats.OfModule(bm); !seat.ByKind || (imageBuilder != nil && imageBuilder()) { out = append(out, bm) } } @@ -191,7 +199,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string if err != nil { err = fmt.Errorf("%s: whether the artifact store still holds %s cannot be asked, so the key is "+ "left done (the controller's take-in refuses it if one is gone): %w", key, strings.Join(refs, ", "), err) - if len(publishable(cur, opt.ImageBuilder)) > 0 { + if len(publishable(cur, opt.Seats, opt.ImageBuilder)) > 0 { return &Claim{Key: key, Rev: rev, Build: cur, Unasked: err}, nil } return nil, err @@ -207,7 +215,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string } } } - if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 { + if opt.Trunk && len(publishable(cur, opt.Seats, opt.ImageBuilder)) > 0 { return &Claim{Key: key, Rev: rev, Build: cur}, nil } return nil, nil diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index bb9b1999..fe5c300c 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -13,11 +13,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" ) -// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that -// run it. It has no toolchain language; the builder finds its nodes from the node-engine's have. -const ImageBuildModule = "mesh-build-image" - -// NoImageBuilderPrefix opens the error of a build key failed because no node runs ImageBuildModule (ADR 0487 clause 5): such a +// NoImageBuilderPrefix opens the error of a build key failed because no node runs the image build module (ADR 0487 clause 5): such a // key is not built again as a new revision, only when a node reports the module (ClaimOptions.ImageBuilder) or the fingerprint // changes. const NoImageBuilderPrefix = "no image builder:" @@ -82,34 +78,55 @@ func (n *NotPinned) Error() string { return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", builds.NotPinned, n.Artifact, n.From) } +// NoBuildModule is the error of an artifact whose build seat or its holder the seat list and the catalog half do not name (yet): +// the plan is not kept, so it is made again when they are read (design 47 G A-82). +type NoBuildModule struct { + Artifact string + Err error +} + +func (n *NoBuildModule) Error() string { return fmt.Sprintf("artifact %s: %v", n.Artifact, n.Err) } +func (n *NoBuildModule) Unwrap() error { return n.Err } + // PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2, ADR 0498 clauses 4 and 5): a Go bundle and an archive are // built by the Go build module, an image by the image build module (ADR 0487), which also copies an upstream image pinned // by its digest, and a TypeScript bundle or package by the Node one, each in the order of the artifacts' names. A manifest // with an upstream artifact pinned by no digest, or a bundle or a package of another language, or a kind no build module // builds, is refused as a whole, naming the first such artifact by name. A manifest with no artifact has an empty plan. -func PlanOf(m catalogue.Manifest) (Plan, error) { +func PlanOf(m catalogue.Manifest, seats *buildrun.BuildSeats) (Plan, error) { if m.Build == nil { return Plan{}, nil } artifacts := append([]catalogue.Artifact(nil), m.Build.Artifacts...) sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) p := Plan{Series: m.Version} + task := func(a catalogue.Artifact, language, kind string) error { + bm, err := seats.For(language, kind) + if err != nil { + return &NoBuildModule{Artifact: a.Name, Err: err} + } + t := Task{Artifact: a.Name, Kind: a.Kind, Module: bm.Module, Language: bm.Toolchain, From: a.From} + if a.Kind != catalogue.ArtifactUpstream { + t.From = "" + } + p.Tasks = append(p.Tasks, t) + return nil + } for _, a := range artifacts { + var err error switch { case a.Kind == catalogue.ArtifactArchive: - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"}) - case a.Kind == catalogue.ArtifactBundle && a.Language == "go": - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"}) - case a.Kind == catalogue.ArtifactBundle && a.Language == "typescript": - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) + err = task(a, "go", a.Kind) // an archive is packed by the Go build seat's holder, which the indexer gives no language + case a.Kind == catalogue.ArtifactBundle && (a.Language == "go" || a.Language == "typescript"): + err = task(a, a.Language, a.Kind) case a.Kind == catalogue.ArtifactImage: - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule}) + err = task(a, "", "image") case a.Kind == catalogue.ArtifactUpstream && !strings.Contains(a.From, "@sha256:"): return Plan{}, &NotPinned{Artifact: a.Name, From: a.From} case a.Kind == catalogue.ArtifactUpstream: - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule, From: a.From}) + err = task(a, "", "image") case a.Kind == catalogue.ArtifactPackage && a.Language == "typescript": - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) + err = task(a, a.Language, a.Kind) case a.Kind == catalogue.ArtifactBundle: return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)} case a.Kind == catalogue.ArtifactPackage: @@ -117,6 +134,9 @@ func PlanOf(m catalogue.Manifest) (Plan, error) { default: return Plan{}, &Refused{Artifact: a.Name, What: a.Kind} } + if err != nil { + return Plan{}, err + } } return p, nil } diff --git a/modules/mesh-delivery-builder/internal/build/plan_test.go b/modules/mesh-delivery-builder/internal/build/plan_test.go index 897f3e1c..a1ef8b83 100644 --- a/modules/mesh-delivery-builder/internal/build/plan_test.go +++ b/modules/mesh-delivery-builder/internal/build/plan_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatdef" ) // manifestOf is a manifest holding these artifacts: the plan reads only their name, kind and language. @@ -22,14 +23,14 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}, catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"}, catalogue.Artifact{Name: "files", Kind: "archive"}) - p, err := PlanOf(m) + p, err := PlanOf(m, testSeats()) if err != nil { t.Fatal(err) } want := []Task{ - {Artifact: "files", Kind: "archive", Module: buildrun.GoBuildModule, Language: "go"}, - {Artifact: "tool", Kind: "bundle", Module: buildrun.GoBuildModule, Language: "go"}, - {Artifact: "web", Kind: "bundle", Module: buildrun.NodeBuildModule, Language: "typescript"}, + {Artifact: "files", Kind: "archive", Module: testGo, Language: "go"}, + {Artifact: "tool", Kind: "bundle", Module: testGo, Language: "go"}, + {Artifact: "web", Kind: "bundle", Module: testNode, Language: "typescript"}, } if len(p.Tasks) != len(want) { t.Fatalf("tasks = %+v, want %+v", p.Tasks, want) @@ -39,7 +40,7 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 t.Errorf("task %d = %+v, want %+v", i, p.Tasks[i], want[i]) } } - if modules := p.Modules(); len(modules) != 2 || modules[0] != buildrun.GoBuildModule || modules[1] != buildrun.NodeBuildModule { + if modules := p.Modules(); len(modules) != 2 || modules[0] != testGo || modules[1] != testNode { t.Errorf("modules = %v", modules) } } @@ -47,24 +48,24 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 const pinned = "nats:2.11.17-alpine@sha256:3333333333333333333333333333333333333333333333333333333333333333" func TestPlanSendsAPinnedUpstreamToTheImageBuildModulesCopyAndRefusesAnUnpinnedOne(t *testing.T) { // B3, ADR 0498 clause 4 - p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned})) + p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned}), testSeats()) if err != nil || len(p.Tasks) != 2 { t.Fatalf("plan %+v, error %v", p, err) } - if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != ImageBuildModule || up.From != pinned || up.Language != "" { - t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, ImageBuildModule) + if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != testImage || up.From != pinned || up.Language != "" { + t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, testImage) } - if p.NeedsToolchain(ImageBuildModule) { + if p.NeedsToolchain(testImage) { t.Error("a copy needs no toolchain") } - _, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"})) + _, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}), testSeats()) if err == nil || !strings.HasPrefix(err.Error(), builds.NotPinned+"up") || !builds.FailedUntilChanged(err.Error()) { t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, builds.NotPinned) } } func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 clause 5 - _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"})) + _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"}), testSeats()) var r *Refused if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), builds.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") { t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, builds.RefusedKind) @@ -72,10 +73,10 @@ func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 } func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) { - if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), builds.RefusedKind) { + if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"}), testSeats()); err == nil || !strings.HasPrefix(err.Error(), builds.RefusedKind) { t.Errorf("a python bundle: error = %v, want a refusal", err) } - p, err := PlanOf(manifestOf()) + p, err := PlanOf(manifestOf(), testSeats()) if err != nil || len(p.Tasks) != 0 { t.Errorf("no artifact: plan %+v error %v, want an empty plan", p, err) } @@ -84,14 +85,36 @@ func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing. func asRefused(err error, target **Refused) bool { return errors.As(err, target) } func TestPlanSendsAnImageToMeshBuildImageWithNoToolchain(t *testing.T) { // 1705a, ADR 0487 clause 5 - p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "img", Kind: "image"}, catalogue.Artifact{Name: "files", Kind: "archive"})) + p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "img", Kind: "image"}, catalogue.Artifact{Name: "files", Kind: "archive"}), testSeats()) if err != nil || len(p.Tasks) != 2 { t.Fatalf("plan %+v, error %v", p, err) } - if p.Tasks[1].Artifact != "img" || p.Tasks[1].Module != ImageBuildModule { - t.Errorf("task = %+v, want the image on %s", p.Tasks[1], ImageBuildModule) + if p.Tasks[1].Artifact != "img" || p.Tasks[1].Module != testImage { + t.Errorf("task = %+v, want the image on %s", p.Tasks[1], testImage) } - if p.NeedsToolchain(ImageBuildModule) { + if p.NeedsToolchain(testImage) { t.Error("an image needs no toolchain") } } + +// A plan's task names the holder of the build seat the seat list and the catalog half give, and a plan made before they name one +// is no refusal and is not kept (design 47 G A-82). +func TestAPlanNamesTheHoldersOfTheBuildSeats(t *testing.T) { + m := manifestOf(catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}, catalogue.Artifact{Name: "img", Kind: "image"}) + defs := map[string]seatdef.Definition{ + "node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}}, + "node-build-image": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: seatdef.OnKindImage}}}, + } + holders := map[string]string{} + seats := buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(s string) (string, bool) { h, ok := holders[s]; return h, ok }) + _, err := PlanOf(m, seats) + var none *NoBuildModule + if !errors.As(err, &none) { + t.Fatalf("with no holder: %v, want NoBuildModule", err) + } + holders["node-build-go"], holders["node-build-image"] = "acme-go", "acme-img" + p, err := PlanOf(m, seats) + if err != nil || len(p.Tasks) != 2 || p.Tasks[0].Module != "acme-img" || p.Tasks[1].Module != "acme-go" || p.Tasks[1].Language != "go" || p.Tasks[0].Language != "" { + t.Fatalf("plan = %+v, %v", p, err) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index 8d7d8a1d..73239fd5 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -42,12 +42,12 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes } func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1 - p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}}) - if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) { - t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(buildrun.GoBuildModule), err) + p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}}, testSeats()) + if err != nil || !p.NeedsToolchain(testGo) { + t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(testGo), err) } - p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}}) - if p.NeedsToolchain(buildrun.GoBuildModule) { + p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}}, testSeats()) + if p.NeedsToolchain(testGo) { t.Error("a plan of an archive needs a toolchain") } } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index b0e4e4c7..b10dcfac 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -41,6 +41,8 @@ type ToolchainEntries interface { // change of the toolchains. type Builder struct { Claimer Claimer + // Seats finds the build modules: the holders of the build seats on the seat list and the catalog half (design 47 G A-82). + Seats *buildrun.BuildSeats // Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq // ADR 0525 §1, §3, §6; issue 1887 A4): the builder chooses no node, whichever build node pulls the ask builds it. Asks interface { @@ -499,7 +501,11 @@ func (b *Builder) planOf(ctx context.Context, digest string) (Plan, error, bool) if err != nil { return Plan{}, fmt.Errorf("its manifest %s cannot be read: %w", digest, err), false } - plan, err := PlanOf(m) + plan, err := PlanOf(m, b.Seats) + var none *NoBuildModule + if errors.As(err, &none) { + return plan, err, false // the seat list or the catalog half names no build module yet: made again when they do + } b.mu.Lock() b.plans[digest] = planned{plan, err} b.mu.Unlock() @@ -575,6 +581,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB defer b.handEnded(key) defer b.unrelease(key) plan, planErr, read := b.planOf(ctx, m.Manifest) + var noModule *NoBuildModule + if errors.As(planErr, &noModule) { + // The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again + // when they do (Reconsider, at each change of either; design 47 G A-82). + b.logf("%s at %s is not built yet: %v", module, m.Fp2, planErr) + return + } nothing := read && planErr == nil && len(plan.Tasks) == 0 if planErr == nil && !nothing && !b.isReleased(key) { if why := b.stale(m, plan); why != "" { @@ -617,7 +630,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write // (issue 1966) is the hand tool's to finish. - cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, + cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { b.logf("%s: %v", module, err) @@ -643,7 +656,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB if cl.Build.State == builds.BuildDone { prior, only = &cl.Build, map[string]bool{} if trunk { - for _, bm := range publishable(cl.Build, imageSeat) { + for _, bm := range publishable(cl.Build, b.Seats, imageSeat) { only[bm] = true publishing = true } @@ -744,7 +757,7 @@ func (b *Builder) stale(m meshcatalog.ModulesByModuleValue, plan Plan) string { if !plan.NeedsToolchain(bm) { continue // an archive is packed with no toolchain } - language := buildrun.LanguageOf[bm] + language, _ := b.Seats.LanguageOf(bm) ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language)) if ag.Wait != "" { return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait) @@ -798,9 +811,9 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module continue } var toolchain *builds.ToolchainUsed - if bm != ImageBuildModule && plan.NeedsToolchain(bm) { + if plan.NeedsToolchain(bm) { // The ask names the toolchain the fingerprint was computed with; a node building with another refuses it. - language := buildrun.LanguageOf[bm] + language, _ := b.Seats.LanguageOf(bm) ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language)) if ag.Wait != "" { return failed("the %s toolchain is not agreed: %s", language, ag.Wait) @@ -910,8 +923,9 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module } for _, bm := range kept { steps = append(steps, bm+" kept") + kept, _ := b.Seats.LanguageOf(bm) for lang, u := range prior.Toolchains { - if _, ok := used[lang]; !ok && buildrun.LanguageOf[bm] == lang { + if _, ok := used[lang]; !ok && kept == lang { used[lang] = u } } @@ -990,9 +1004,6 @@ func keyError(errs []string) string { // want of an image builder before R4g is taken again (ADR 0525 §1). func imageSeat() bool { return true } -// askLanguage is the build seat language of each build module (sdkstep.AskSubject). -var askLanguage = map[string]string{buildrun.GoBuildModule: sdkstep.AskGo, buildrun.NodeBuildModule: sdkstep.AskNode, ImageBuildModule: sdkstep.AskImage} - // askNotes writes a claimed key's asks into it, building, before each is put (ADR 0525 §1; issue 1887 A4b), and holds // the asks a key taken over at start already names, so they are waited for and never put again. type askNotes struct { @@ -1062,7 +1073,8 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes sort.Strings(req.Paths) } candidate := req.Base != "" - language := askLanguage[l.module] + seat, _ := b.Seats.OfModule(l.module) + language := seat.AskLanguage() kind := sdkstep.AskDelivery if candidate { kind = sdkstep.AskCheck diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index ce3d66f3..3d3303d0 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -145,7 +145,7 @@ type rig struct { } // toolPrefix is the start of each build module's tool names, as the fake build modules answer them. -var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"} +var toolPrefix = map[string]string{testGo: "build_go_", testNode: "build_node_", testImage: "build_image_"} // answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the // rig's pulling node, and answers the step that node writes. @@ -178,14 +178,14 @@ func (r *rig) answer(a buildstep.StepAsk) buildstep.Step { func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { t.Helper() - r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} + r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{Seats: testSeats()}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} for _, n := range []string{"ace", "novox"} { r.tools.Set(n, tcOf("go", "go1.27.1")) r.tools.Set(n, tcOf("typescript", "tsc5")) } r.asks = &buildrun.FakeAsks{Answer: r.answer} r.b = &Builder{ - Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second}, + Seats: testSeats(), Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second}, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf, Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil }, } @@ -660,7 +660,7 @@ func aNewerFingerprintWithdrawsTheBuildOfTheOlderOne(t *testing.T) { } return r.answer(a) } - r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second, Dropper: drops} + r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second, Dropper: drops} r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) <-running fp2 := "src2:" + hex64('2') @@ -908,7 +908,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { t.Run("waits in its stream: never asked again", func(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil - r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} + r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} gone := &goneAsks{why: func(buildstep.StepAsk) string { return "" }} r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) @@ -1115,12 +1115,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { gone := &goneAsks{} dead := &buildrun.DeadLetters{} if c.dead { - r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second} + r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second} } var heard *buildrun.HeardEnds if c.deaf { heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks} - r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second} + r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second} } if !c.byHand && !c.dead && !c.deaf { gone.why = func(a buildstep.StepAsk) string { @@ -1151,7 +1151,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { t.Fatal("no ask is waited on") } } - heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended + heard.Nudge(testGo) // a step of mesh-build-go ended } if c.done && !c.deaf { asks := waitForAsks(t, r, c.puts) @@ -1262,7 +1262,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { running, _ := json.Marshal(buildstep.Step{V: keys.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: buildstep.StepRunning, Started: at}) _, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at) r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}), - Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})}, + Seats: testSeats(), Steps: map[string]StateRead{"node-build-go": state(map[string]string{}), "node-build-image": state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})}, Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}), Now: func() time.Time { return at.Add(time.Hour) }} out, err := r.Stuck("") diff --git a/modules/mesh-delivery-builder/internal/build/seats_test.go b/modules/mesh-delivery-builder/internal/build/seats_test.go new file mode 100644 index 00000000..71fad1fb --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/seats_test.go @@ -0,0 +1,23 @@ +package build + +import ( + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82). +const ( + testGo = "mesh-build-go" + testNode = "mesh-build-node" + testImage = "mesh-build-image" +) + +// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module. +func testSeats() *buildrun.BuildSeats { + rule := func(on string) seatdef.Definition { + return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}} + } + defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)} + holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage} + return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }) +} diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 560bce85..988e398c 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -239,8 +239,13 @@ type Stuck struct { type StuckReader struct { Builds StateRead ReAsks StateRead - // Steps are the build modules' steps states, by build module. + // Steps are the build seats' steps states (`.steps`, design 47 G A-82), by build seat. Steps map[string]StateRead + // StepsOf, when set, reads the steps state of a build seat the seat list names (`.steps`), instead of Steps: the seats + // are known only once the list is read. + StepsOf func(seat string) StateRead + // Seats finds the build module of a build seat, whose name the key's step words give. + Seats *buildrun.BuildSeats // Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said. Hearing StateRead Now func() time.Time @@ -318,7 +323,14 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { } // Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it. stepKeys := map[string]map[string]string{} - for bm, s := range r.Steps { + steps := r.Steps + if r.StepsOf != nil { + steps = map[string]StateRead{} + for _, m := range r.Seats.All() { + steps[m.Seat] = r.StepsOf(m.Seat) + } + } + for bm, s := range steps { ks, err := s.Keys() if err != nil { return nil, fmt.Errorf("the steps of %s: %w", bm, err) @@ -385,13 +397,14 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { s.Age = now().Sub(at).Round(time.Second).String() } for language, id := range v.Asks { - bm, _ := buildstep.BuildModuleOf(language) - a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"} + seat, _ := buildstep.BuildSeat(language) + bm, _ := r.Seats.OfSeat(seat) + a := StuckAsk{Language: language, BuildModule: bm.Module, ID: id, Asking: askingOf(v.Step, bm.Module), Step: "none"} if t, ok := idTime(id); ok { a.Asked = t } - if st, ok := r.Steps[bm]; ok { - if k, ok := stepKeys[bm][id]; ok { + if st, ok := steps[seat]; ok { + if k, ok := stepKeys[seat][id]; ok { reads++ if raw, _, err := st.Get(k); err == nil && raw != nil { if step, err := buildstep.DecodeStep(raw); err == nil { @@ -432,9 +445,6 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { return answer, nil } -// StuckBuildModules are the build modules whose steps builder_stuck reads. -var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} - // idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits). func idTime(id string) (time.Time, bool) { if len(id) <= 6 { diff --git a/modules/mesh-delivery-builder/internal/build/waits.go b/modules/mesh-delivery-builder/internal/build/waits.go index bf3ca02d..2c1f2cf5 100644 --- a/modules/mesh-delivery-builder/internal/build/waits.go +++ b/modules/mesh-delivery-builder/internal/build/waits.go @@ -9,7 +9,6 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) @@ -57,7 +56,7 @@ func (b *Builder) toolchainWait(module string, m meshcatalog.ModulesByModuleValu if !plan.NeedsToolchain(bm) { continue } - language := buildrun.LanguageOf[bm] + language, _ := b.Seats.LanguageOf(bm) ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language)) if want, ok := m.Toolchains[language]; ag.Wait == "" && ok && want.Version != ag.Toolchain.Version { return ToolchainWait{Module: module, Fingerprint: m.Fp2, Language: language, Version: want.Version, Since: b.now()}, true diff --git a/modules/mesh-delivery-builder/internal/versions/entrynames.go b/modules/mesh-delivery-builder/internal/versions/entrynames.go index a140613f..a4c4ffa0 100644 --- a/modules/mesh-delivery-builder/internal/versions/entrynames.go +++ b/modules/mesh-delivery-builder/internal/versions/entrynames.go @@ -2,11 +2,15 @@ package versions import ( "context" + "encoding/json" + "slices" "sort" "strings" + "sync" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // Entries are the catalog graph entries of the halves the builder follows, one per module and watched branch (design 47 G §348, @@ -80,3 +84,54 @@ func EntriesOf(branches *meshcatalogclient.Branches) Entries { return out } } + +// HolderOf finds the module that holds a seat: the one whose entry in the first followed branch's half (the trunk's) has a +// manifest that claims it (design 47 G A-82; the seat list answers seat definitions alone, §527). More than one claimant, or +// none, is no holder. +func HolderOf(branches *meshcatalogclient.Branches) func(seat string) (string, bool) { + var mu sync.Mutex + claims := map[string][]string{} // manifest digest -> the seats it claims + claimed := func(e meshcatalog.CatalogEntry) []string { + mu.Lock() + defer mu.Unlock() + if seats, ok := claims[e.ManifestDigest]; ok && e.ManifestDigest != "" { + return seats + } + var seats []string + if raw, err := json.Marshal(e.Manifest); err == nil { + if mf, err := catalogue.Decode(raw); err == nil { + for _, c := range mf.Claims { + seats = append(seats, c.Name) + } + } + } + if e.ManifestDigest != "" { + claims[e.ManifestDigest] = seats + } + return seats + } + return func(seat string) (string, bool) { + followed := branches.Followed() + if len(followed) == 0 { + return "", false + } + cat, ok := branches.Catalog(followed[0]) + if !ok { + return "", false + } + half, ok := cat.View() + if !ok { + return "", false + } + var holders []string + for _, e := range half.Entries { + if slices.Contains(claimed(e), seat) { + holders = append(holders, e.Module) + } + } + if len(holders) != 1 { + return "", false + } + return holders[0], true + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/floor.go b/modules/mesh-delivery-builder/internal/versions/floor.go index 8beea79b..516e7776 100644 --- a/modules/mesh-delivery-builder/internal/versions/floor.go +++ b/modules/mesh-delivery-builder/internal/versions/floor.go @@ -77,11 +77,9 @@ const ( // sourceOfWake is the floor source each event says something of; a clear names the source by its module. var sourceOfWake = map[string]string{WakePublished: SourceSeat, WakeForge: SourceTags, WakeAccepted: SourceRequirements} -// ModuleOfSource is the module whose health condition's clear wakes a silent source. -var ModuleOfSource = map[string]string{ - SourceTags: "mesh-delivery-forge-bridge", - SourceRequirements: "mesh-delivery-indexer", -} +// SourceSeats are the seats whose holders are the floor's tag and requirement sources (design 47 G A-82): the seat git (tags) +// and the seat mesh-catalog (requirements), the holders read from the seat list and the catalog half. +var SourceSeats = map[string]string{SourceTags: "git", SourceRequirements: "mesh-catalog"} // FloorWait is one name waiting on its floor: the sources still silent and why, since when; the finding the builder // writes names them, cleared once all three answer. @@ -135,6 +133,8 @@ type FloorWaits struct { keeper FloorKeeper // Logf says a wait that could not be kept; nil says nothing. Logf func(string, ...any) + // Holder finds the module holding a seat (the catalog half's claim); a source's module is the holder of its SourceSeats seat. + Holder func(seat string) (string, bool) // Said is told each wait recorded (raised) and each answered (not raised): the builder's finding floor-silent. Said func(w FloorWait, raised bool) } @@ -290,7 +290,7 @@ func (fw *FloorWaits) Wake(kind, seat, module, name string) []string { } case WakeCleared: for _, s := range w.Silent { - if ModuleOfSource[s] == module || (s == SourceSeat && w.Holder != "" && w.Holder == module) { + if fw.holds(SourceSeats[s], module) || (s == SourceSeat && w.Holder != "" && w.Holder == module) { out = append(out, n) break } @@ -317,3 +317,12 @@ func (fw *FloorWaits) Waiting() []FloorWait { sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out } + +// holds is whether module holds seat on the catalog half. +func (fw *FloorWaits) holds(seat, module string) bool { + if seat == "" || module == "" || fw.Holder == nil { + return false + } + holder, ok := fw.Holder(seat) + return ok && holder == module +} diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index 59b9b7a3..ebb7bbbc 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -251,7 +251,8 @@ func TestTheBuildersVersions(t *testing.T) { if f.Known() || !reflect.DeepEqual(f.Silent, []string{SourceTags}) || why[SourceTags] != "no answer" { t.Fatalf("Ask = %+v, %v; want bridge_tags silent", f, why) } - var fw FloorWaits + holders := map[string]string{SourceSeats[SourceTags]: "mesh-delivery-forge-bridge", SourceSeats[SourceRequirements]: "mesh-delivery-indexer"} + fw := FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }} w := fw.Wait("m/program", "oci-artifact-registry", "distribution", f, why, time.Unix(0, 0)) if !strings.Contains(w.Finding(), "bridge_tags (no answer)") { t.Errorf("finding = %q", w.Finding()) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 075fe81e..6914f076 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -118,8 +118,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a // node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3). - if want := []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", - "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-delivery-assigner.assignments", "mesh-delivery-gate.source", + "node-build-go.steps", "node-build-image.steps", "node-build-node.steps"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -140,7 +140,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). - if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.remove", "seat:container-image-registry.versions", + if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) { @@ -154,7 +154,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // DLV-M1: the indexer's and the assigner's views families of the states it follows. if want := []string{"container-image-registry.kept", "container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed", "go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed", - "mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", + "mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-forge-bridge.forge.**", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", @@ -169,7 +169,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 6f0808a6..49972d01 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -13,6 +13,8 @@ ], "tools": [ "builder_announce_again", + "builder_seat_list", + "builder_seat_list_again", "builder_ask_again", "builder_bounds", "builder_build_again", @@ -51,6 +53,8 @@ "go-package-registry.removed", "mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", + "mesh-catalog.seat-definition-changed", + "mesh-catalog.seat-definition-removed", "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", @@ -96,6 +100,7 @@ "mesh-delivery-assigner.assigner_read_assignments", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", + "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source" ], @@ -134,9 +139,9 @@ { "name": "build-log", "max-age-days": 30, "max-mib": 3072 } ], "reads": [ - "mesh-build-go.steps", - "mesh-build-image.steps", - "mesh-build-node.steps", + "node-build-go.steps", + "node-build-image.steps", + "node-build-node.steps", "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source" ], diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/buildmodules.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/buildmodules.go new file mode 100644 index 00000000..f07a698c --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/buildmodules.go @@ -0,0 +1,23 @@ +package main + +import ( + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The checker's names of the build modules, kept until its part finds them on the seat list as the builder does (design 47 G +// A-82 is the builder's change; the builder's package holds no name of them any more): the holders of the three build seats. +const ( + goBuildModule = "mesh-build-go" + nodeBuildModule = "mesh-build-node" +) + +// checkerBuildSeats is the finder over the build seats those names hold. +var checkerBuildSeats = func() *buildrun.BuildSeats { + rule := func(on string) seatdef.Definition { + return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}} + } + defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript")} + holders := map[string]string{"node-build-go": goBuildModule, "node-build-node": nodeBuildModule} + return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }) +}() diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 21a4f63b..509aab7c 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -404,7 +404,7 @@ func (r *repoModules) PackagePaths(repository string) []string { // is judged by both, each finding its own directories in the tree (go.mod files; package.json files), so a repository // with tests in either language and no manifest is judged whole (issue 1887 A12). func (r *repoModules) BuildModules(repository string) []string { - out := []string{buildrun.GoBuildModule} + out := []string{goBuildModule} r.mu.Lock() defer r.mu.Unlock() known := false @@ -414,12 +414,12 @@ func (r *repoModules) BuildModules(repository string) []string { } known = true if r.langs[r.typed[module]]["typescript"] { - out = append(out, buildrun.NodeBuildModule) + out = append(out, nodeBuildModule) break } } if !known { - out = append(out, buildrun.NodeBuildModule) + out = append(out, nodeBuildModule) } sort.Strings(out) return out diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 14d3f199..a42ad2ab 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/keys" @@ -70,10 +69,10 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test r := &repoModules{manifests: store} putModule(t, r, "mesh-delivery-checker", "novox/mesh-delivery", goManifest) putModule(t, r, "mesh-ui", "novox/mesh-catalog", tsManifest) - if got := r.BuildModules("novox/mesh-delivery"); strings.Join(got, ",") != buildrun.GoBuildModule { + if got := r.BuildModules("novox/mesh-delivery"); strings.Join(got, ",") != goBuildModule { t.Errorf("mesh-delivery: %v, want the Go build module alone", got) } - if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule { + if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != goBuildModule+","+nodeBuildModule { t.Errorf("mesh-catalog: %v, want both", got) } if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" { @@ -81,7 +80,7 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test } // 1887/A12: a repository with no module the mesh knows (mesh-lab: nested go.mod files and a root package.json) is // judged by both build modules. - if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule { + if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != goBuildModule+","+nodeBuildModule { t.Errorf("a repository with no module: %v, want both build modules", got) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 1d6548c4..7c37d5a1 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -210,7 +210,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if link == "" { link = defaultLink } - tools := &buildrun.ToolchainSet{} + tools := &buildrun.ToolchainSet{Seats: checkerBuildSeats} presence := &buildrun.PresenceSet{} trunks := &check.Trunks{} // A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a @@ -228,7 +228,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer hearing.Stop() // Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read // (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28). - heard := buildrun.HearEnds(ctx, asks, asks, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, logger.Printf) + heard := buildrun.HearEnds(ctx, asks, asks, []string{goBuildModule, nodeBuildModule}, logger.Printf) conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") }) waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing} asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf) @@ -285,7 +285,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Asker: engineAsks{call: caller.CallOn}, Placed: placedNodes{kv: func(context.Context) (jetstream.KeyValue, error) { return assignments, nil }}.Placed, Bound: answerBound, - Sandbox: seatSandbox{asker: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}}, + Sandbox: seatSandbox{asker: buildrun.Asker{Seats: checkerBuildSeats, Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}}, Verdicts: verdicts.all, // The node-engine's trunk head: a group with no node-engine member composes with the composer built there (ADR 0656 §5). Composer: func() string { return trunks.Heads()[check.NodeEngineRepository] }, @@ -314,7 +314,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6) ch.Says = verdictSays{js: js, node: cred.Node, now: time.Now} // a pull request's verdict event, on the stream asks (design 26 *Out of step*) ch.Judge = newJudge(judgeDeps{ - asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}, tools: tools, trunks: trunks, + asks: buildrun.Asker{Seats: checkerBuildSeats, Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}, tools: tools, trunks: trunks, kept: kvStore{buckets[check.CheckerKeptBucket]}, previews: kvStore{buckets[check.PreviewsBucket]}, candidates: tryCandidates{candidates}, repos: repos, published: published, builds: buildStates, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf, announce: announce, compositions: compositions, inputs: runInputs{awaitedKV{inputsKV}}, beginBound: beginBound, @@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger) // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the // assignments is settled first, so the toolchains settle through the gate already knowing it. - gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) + gate := buildrun.NewToolchainGate([]string{goBuildModule, nodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain // Until the assignments's first view is answered the gate passes every report on, as before the assignments is settled. assignmentsErrs := make(chan error, 1) @@ -354,7 +354,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A build module says each time it wrote its toolchains key: its stream, and the checker's durable consumer on it, are // made by the controller. Where the stream is not there yet the checker says so and is started again with the build // module. - for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} { + for _, module := range []string{goBuildModule, nodeBuildModule} { stream := toolchainStreams[module] cons, err := cred.EventConsumer(ctx, js, stream) if err != nil { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go index 9b980baa..304cbd30 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go @@ -20,7 +20,7 @@ import ( // its grant. The gate over the seat's assignments, and its refusal, are mesh-sdk's (buildrun.ToolchainGate, // issue 774). func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { - set := &buildrun.ToolchainSet{} + set := &buildrun.ToolchainSet{Seats: checkerBuildSeats} var logs bytes.Buffer logger := log.New(&logs, "", 0) refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go index 7ac53a3b..eb75a3c2 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go @@ -17,7 +17,7 @@ import ( // toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it // (declared by the build modules, heard by this module through consumes). -var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO", buildrun.NodeBuildModule: "TOOLCHAINS_NODE"} +var toolchainStreams = map[string]string{goBuildModule: "TOOLCHAINS_GO", nodeBuildModule: "TOOLCHAINS_NODE"} // toolchainWatch keeps the build modules' toolchains in a buildrun.ToolchainSet and tells the builder when one changed. A // build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437 @@ -37,12 +37,12 @@ type toolchainWatch struct { } func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch { - return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}} + return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(checkerBuildSeats.Modules())), watched: map[string]bool{}} } // start watches the buckets that exist now; a missing one is said and waits for its build module's event. func (w *toolchainWatch) start() error { - for module := range buildrun.LanguageOf { + for _, module := range checkerBuildSeats.Modules() { if err := w.ensure(module); err != nil { return err } @@ -96,7 +96,7 @@ type delivered interface { // terminated; a failure stays unacknowledged, and the server delivers it again. func (w *toolchainWatch) Reported(m delivered) { module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.") - if _, known := buildrun.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") { + if _, known := checkerBuildSeats.LanguageOf(module); !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") { w.logger.Printf("%s: not a build module's event, dropped", m.Subject()) _ = m.Term() return diff --git a/modules/mesh-delivery-checker/internal/check/build_change.go b/modules/mesh-delivery-checker/internal/check/build_change.go index 998e9f98..17f7cbee 100644 --- a/modules/mesh-delivery-checker/internal/check/build_change.go +++ b/modules/mesh-delivery-checker/internal/check/build_change.go @@ -7,7 +7,6 @@ import ( "slices" "strings" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -75,13 +74,13 @@ func (st *state) buildsOfTheCheck(ctx context.Context) (map[string]string, []str if !readable { return wait, words } - for _, bm := range slices.Sorted(maps.Keys(buildrun.LanguageOf)) { + for _, bm := range slices.Sorted(maps.Keys(languageOf)) { if _, alters := changed[bm]; !alters { continue } var built []string for module, m := range changed { - if _, isBuildModule := buildrun.LanguageOf[module]; !isBuildModule && buildsWith(m, buildrun.LanguageOf[bm]) { + if _, isBuildModule := languageOf[module]; !isBuildModule && buildsWith(m, languageOf[bm]) { built = append(built, module) delete(wait, module) } diff --git a/modules/mesh-delivery-checker/internal/check/buildmodules.go b/modules/mesh-delivery-checker/internal/check/buildmodules.go new file mode 100644 index 00000000..9ce5ca9c --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/buildmodules.go @@ -0,0 +1,11 @@ +package check + +// The checker's names of the build modules, kept until its part finds them on the seat list as the builder does (design 47 G +// A-82 is the builder's change; the builder's package holds no name of them any more). +const ( + goBuildModule = "mesh-build-go" + nodeBuildModule = "mesh-build-node" +) + +// languageOf is the language each build module's toolchain reports hold. +var languageOf = map[string]string{goBuildModule: "go", nodeBuildModule: "typescript"} diff --git a/modules/mesh-delivery-checker/internal/check/buildseats_test.go b/modules/mesh-delivery-checker/internal/check/buildseats_test.go new file mode 100644 index 00000000..e2eafa80 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/buildseats_test.go @@ -0,0 +1,16 @@ +package check + +import ( + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// testSeats is the finder over the build seats the checker's names hold (buildmodules.go). +func testSeats() *buildrun.BuildSeats { + rule := func(on string) seatdef.Definition { + return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}} + } + defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript")} + holders := map[string]string{"node-build-go": goBuildModule, "node-build-node": nodeBuildModule} + return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }) +} diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go index 592a6e5c..b5efb00e 100644 --- a/modules/mesh-delivery-checker/internal/check/checker_test.go +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -450,7 +450,7 @@ func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testi done := make(chan struct{}) go func() { defer close(done) - (buildrun.Asker{Publisher: fake, Waiter: fake, Bound: time.Hour, Dropper: drops}).Wait(run, buildstep.StepAsk{ID: "mvl0frv18d2000", Language: buildstep.AskGo, Kind: buildstep.AskCheck}) + (buildrun.Asker{Seats: testSeats(), Publisher: fake, Waiter: fake, Bound: time.Hour, Dropper: drops}).Wait(run, buildstep.StepAsk{ID: "mvl0frv18d2000", Language: buildstep.AskGo, Kind: buildstep.AskCheck}) }() ch.Forget("g") <-done diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 80d166a4..755d7339 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -106,7 +106,7 @@ func (l langs) BuildModules(repository string) []string { if m, ok := l[repository]; ok { return m } - return []string{buildrun.GoBuildModule} + return []string{goBuildModule} } func goEntries(nodes ...string) entries { @@ -207,7 +207,7 @@ func refsOf(v any) []BesideRef { } // toolPrefix is the start of each build module's tool names, as the fake build modules answer them. -var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"} +var toolPrefix = map[string]string{goBuildModule: "build_go_", nodeBuildModule: "build_node_"} // answer runs a check ask as the build module that pulls it does (gobuild.RunAsk), with the fake tools on shanks and the // bench's kept verdicts, and answers the step it writes. @@ -228,7 +228,7 @@ func (b *bench) answer(a buildstep.StepAsk) buildstep.Step { k, found, err := (Kept{Store: b.kept}).Lookup(ctx, level, digest) return int(k.Tests), found, err }} - if module == buildrun.GoBuildModule { + if module == goBuildModule { steps.Digest = tool("digest") } began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) @@ -278,7 +278,7 @@ func newBench(t testing.TB) *bench { } now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) b.asks = &buildrun.FakeAsks{Answer: b.answer} - b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev, + b.ex = &Examiner{Asks: buildrun.Asker{Seats: testSeats(), Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev, Modules: langs{}, Trunks: &Trunks{heads: map[string]string{DeliveryRepository: trunkD}}, Inputs: memInputs{b.inputs}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }} return b } @@ -559,7 +559,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3 func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1292/AC1 b := newBench(t) - b.ex.Modules = langs{repoA: {buildrun.NodeBuildModule}} + b.ex.Modules = langs{repoA: {nodeBuildModule}} b.fetch = captured[Fetched](t, "node-fetch-clean") b.test = func(c stepCall) (any, error) { return Tested{Level: c.args["level"].(string), Module: "modules/m", Passed: true, Elapsed: 0.4, Bound: 5}, nil @@ -991,7 +991,7 @@ func (p pkgPaths) PackagePaths(repository string) []string { return p[repository // groupPackageBench is a group of a TypeScript member (mesh-delivery) and the mesh-sdk member, whose merge changes sdkChanged. func groupPackageBench(t *testing.T, sdkChanged []string) *bench { b := newBench(t) - b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}, repoSDK: {buildrun.GoBuildModule, buildrun.NodeBuildModule}} + b.ex.Modules = langs{repoA: {goBuildModule, nodeBuildModule}, repoSDK: {goBuildModule, nodeBuildModule}} b.ex.Packages = pkgPaths{repoSDK: {"src", "test", "package.json", "package-lock.json", "tsconfig.json"}} fetch := func(c stepCall) (any, error) { f := b.fetch @@ -1006,7 +1006,7 @@ func groupPackageBench(t *testing.T, sdkChanged []string) *bench { func nodeFetchOf(b *bench, repo string) stepCall { for _, c := range b.steps.tool("fetch") { - if c.module == buildrun.NodeBuildModule && c.args["repository"] == repo { + if c.module == nodeBuildModule && c.args["repository"] == repo { return c } } @@ -1065,7 +1065,7 @@ func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) { b.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { return buildstep.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: buildstep.StepRunning, Started: time.Now()} } - b.ex.Asks = buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond} + b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond} out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) if f := out.Verdict.Failed; out.Verdict.State != meshdeliverycheckerclient.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") { t.Fatalf("verdict = %+v", out.Verdict) @@ -1105,7 +1105,7 @@ func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) { b := newBench(t) answer := b.asks.Answer b.asks.Answer = nil - b.ex.Asks = buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond} + b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond} done := make(chan Outcome, 1) go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }() for deadline := time.Now().Add(10 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) { diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 1476eae8..099ba764 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -19,7 +19,7 @@ import ( ) // goModule is the build module the Go repositories are judged by. -const goModule = buildrun.GoBuildModule +const goModule = goBuildModule // RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build // module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut @@ -69,7 +69,7 @@ type BuildModules interface { } // askLanguage is the build seat language of each build module the checker asks. -var askLanguage = map[string]string{buildrun.GoBuildModule: buildstep.AskGo, buildrun.NodeBuildModule: buildstep.AskNode} +var askLanguage = map[string]string{goBuildModule: buildstep.AskGo, nodeBuildModule: buildstep.AskNode} // Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request // and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level @@ -545,17 +545,17 @@ func (st *state) judgeAll(ctx context.Context) { modules := st.j.Modules.BuildModules(p.Repository) for _, bm := range modules { t := &target{pull: p, module: bm} - if tc, none := buildstep.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" { + if tc, none := buildstep.AgreedToolchain(st.j.Toolchains.Entries(languageOf[bm])); none == "" { t.toolchain = tc.Version - t.agreed = &buildstep.ToolchainUsed{Language: buildrun.LanguageOf[bm], Version: tc.Version, Recipe: tc.Recipe} + t.agreed = &buildstep.ToolchainUsed{Language: languageOf[bm], Version: tc.Version, Recipe: tc.Recipe} } args := buildstep.Fetch{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number), Base: p.Trunk} - if bm == buildrun.NodeBuildModule { + if bm == nodeBuildModule { if groupPackage != nil && p.Repository != SDKRepository { args.Package = groupPackage } } - if bm == buildrun.GoBuildModule { + if bm == goBuildModule { beside, err := st.j.besideFor(ctx, st.c, p) if err != nil { st.fail(forgeMachine(err, bm), err.Error()+unheardWords(err)) @@ -797,7 +797,7 @@ func unmetOf(levels []buildstep.LevelAnswer) string { // one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's // running time (its tests' and setup's). func (st *state) level(ctx context.Context, t *target, la buildstep.LevelAnswer) float64 { - if t.runner != "" && t.module == buildrun.GoBuildModule && la.Digest != nil { + if t.runner != "" && t.module == goBuildModule && la.Digest != nil { t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again } if t.runner != "" && la.Judged != "" { @@ -806,7 +806,7 @@ func (st *state) level(ctx context.Context, t *target, la buildstep.LevelAnswer) _, elapsed, results := st.levels.Level(t.pull.Repository, t.pull.Number, t.module, t.node, la) st.facts = append(st.facts, rules.Step{Repository: t.pull.Repository, Number: t.pull.Number, Module: t.module, Node: t.node, Level: rules.LevelFact(la)}) var digests map[string]string - if t.module == buildrun.GoBuildModule && la.Digest != nil { + if t.module == goBuildModule && la.Digest != nil { // The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too. digests = la.Digest.Map() } diff --git a/modules/mesh-delivery-checker/internal/check/lostask_test.go b/modules/mesh-delivery-checker/internal/check/lostask_test.go index a9931fd9..d9464d6e 100644 --- a/modules/mesh-delivery-checker/internal/check/lostask_test.go +++ b/modules/mesh-delivery-checker/internal/check/lostask_test.go @@ -63,7 +63,7 @@ func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data [] func losing(b *bench, lose int) *losingAsks { l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose} - b.ex.Asks = buildrun.Asker{Publisher: b.asker(l, l.FakeAsks.Write), Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second} + b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(l, l.FakeAsks.Write), Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second} return l } @@ -288,7 +288,7 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) { tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf, deafIDs: map[string]bool{}, steps: map[string]buildstep.Step{}} heard = &buildrun.HeardEnds{Inner: tr, Read: tr} - b.ex.Asks = buildrun.Asker{Publisher: b.asker(tr, tr.Write), Waiter: heard, Bound: 5 * time.Second} + b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(tr, tr.Write), Waiter: heard, Bound: 5 * time.Second} asked = func() int { return len(tr.Asks()) } } r := newRig(t, b.ex) @@ -307,23 +307,23 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) { } switch row.how { case "nudge": - heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read - heard.Nudge(buildrun.GoBuildModule) + heard.Nudge(nodeBuildModule) // another build module's step: not this one's to read + heard.Nudge(goBuildModule) case "hand": cands := newMemKV() ckey, _ := meshcatalogclient.CandidatesKey("feat/x") _, _ = cands.Create(ctx, ckey, cand) none := StateRead{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, error) { return nil, nil }} rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept), - StepStates: map[string]StateRead{buildrun.GoBuildModule: none}} // no node took its ask yet + StepStates: map[string]StateRead{goBuildModule: none}} // no node took its ask yet if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "waits in its build seat's stream") { t.Errorf("asked again while its ask waits in the stream (issue 1791's kind): %v", err) } - rd.StepStates[buildrun.GoBuildModule] = tr.Steps(true) + rd.StepStates[goBuildModule] = tr.Steps(true) if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "a step of it runs") { t.Errorf("asked again while its step runs: %v", err) } - rd.StepStates[buildrun.GoBuildModule] = tr.Steps(false) + rd.StepStates[goBuildModule] = tr.Steps(false) if st, _ := rd.Status(); !strings.Contains(st.Lost["feat/x"], AskAgainTool) { t.Errorf("status lost = %v, want feat/x named with %s", st.Lost, AskAgainTool) } @@ -368,7 +368,7 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) { id := fmt.Sprint("mvl0frv18d19", 80+i) vals["ace."+id], _ = buildstep.EncodeStep(buildstep.Step{ID: id, Node: "ace", Tool: "check ask", State: buildstep.StepFailed, Passing: p, Ended: time.Now(), What: "check " + repoA + "@" + short12(head1)}) } - rd := Reader{CandidateState: kvRead(cands2(t, cand)), VerdictState: kvRead(failedKV), AskState: kvRead(newMemKV()), StepStates: map[string]StateRead{buildrun.GoBuildModule: { + rd := Reader{CandidateState: kvRead(cands2(t, cand)), VerdictState: kvRead(failedKV), AskState: kvRead(newMemKV()), StepStates: map[string]StateRead{goBuildModule: { Keys: func() ([]string, error) { return slices.Collect(maps.Keys(vals)), nil }, Get: func(k string) (json.RawMessage, error) { return vals[k], nil }}}} if got, err := rd.AskAgain(asVerdicts{failedKV}, "feat/x", "the network"); (err == nil) != c.ok || (c.ok && !strings.HasPrefix(got.Said, AskedAgainWords)) { t.Errorf("%s: asked again = %+v, %v, want asked again %v", c.name, got, err, c.ok) diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go index e7f6956d..eedb4e7e 100644 --- a/modules/mesh-delivery-checker/internal/check/pins_test.go +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -9,7 +9,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/grouppackages" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -119,7 +118,7 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes b = newBench(t) b.fetch.Modules = []string{"modules/m"} b.ex.Read = fixedManifests{digest: sdk} - b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}} + b.ex.Modules = langs{repoA: {goBuildModule, nodeBuildModule}} if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), sdkPull)).Verdict; v.State != meshdeliverycheckerclient.VerdictPass { t.Errorf("a group's verdict = %+v", v) } diff --git a/modules/mesh-delivery-checker/rules/buildmodules.go b/modules/mesh-delivery-checker/rules/buildmodules.go new file mode 100644 index 00000000..805158a0 --- /dev/null +++ b/modules/mesh-delivery-checker/rules/buildmodules.go @@ -0,0 +1,8 @@ +package rules + +// The checker's names of the build modules, kept until its part finds them on the seat list as the builder does (design 47 G +// A-82 is the builder's change; the builder's package holds no name of them any more). +const ( + goBuildModule = "mesh-build-go" + nodeBuildModule = "mesh-build-node" +) diff --git a/modules/mesh-delivery-checker/rules/facts.go b/modules/mesh-delivery-checker/rules/facts.go index 1a5d7ec4..b9b559cb 100644 --- a/modules/mesh-delivery-checker/rules/facts.go +++ b/modules/mesh-delivery-checker/rules/facts.go @@ -15,7 +15,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) @@ -159,7 +158,7 @@ func (t Tally) Level(repository string, number int, module, node string, la buil } results = append(results, res) elapsed += res.Elapsed + res.Setup + res.Lint - if module == buildrun.NodeBuildModule { + if module == nodeBuildModule { if !res.NoTests { a.Run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10) } diff --git a/modules/mesh-delivery-checker/rules/graph.go b/modules/mesh-delivery-checker/rules/graph.go index 21b8a58c..166ddcc1 100644 --- a/modules/mesh-delivery-checker/rules/graph.go +++ b/modules/mesh-delivery-checker/rules/graph.go @@ -4,8 +4,6 @@ import ( "fmt" "slices" "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" ) // The module graph's cycle rule (novox/hq ADR 0603 §2, ADR 0614 §3, §6), judged by the checking program (ADR 0656 §5): @@ -60,7 +58,7 @@ func JudgeGraph(g Graph) (*FailedFirst, []string) { return nil, []string{"the module graph has the cycle " + words + ", which the trunk already has among the same modules (ADR 0603 §2)"} } } - return &FailedFirst{Machine: buildrun.GoBuildModule, Words: fmt.Sprintf("the change's module graph has the cycle %s: these modules would "+ + return &FailedFirst{Machine: goBuildModule, Words: fmt.Sprintf("the change's module graph has the cycle %s: these modules would "+ "each wait for the other for good, through their bases, the provisions and bus state they await, or the verbs they invoke (ADR 0603 §2, ADR 0614 §6)", words)}, nil } diff --git a/modules/mesh-delivery-checker/rules/judged.go b/modules/mesh-delivery-checker/rules/judged.go index 80b9946a..a5d99b06 100644 --- a/modules/mesh-delivery-checker/rules/judged.go +++ b/modules/mesh-delivery-checker/rules/judged.go @@ -12,7 +12,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/composer" ) @@ -21,7 +20,7 @@ import ( // (milestone 47/B0, D1b): the running checker asks and records; these rules give the verdict. // manifestsMachine is the machine a manifest's failure names: the build seat that ran the manifest validator. -const manifestsMachine = buildrun.GoBuildModule +const manifestsMachine = goBuildModule // CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and // never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4). diff --git a/modules/mesh-delivery-checker/rules/words.go b/modules/mesh-delivery-checker/rules/words.go index bd94a532..9a1d06a6 100644 --- a/modules/mesh-delivery-checker/rules/words.go +++ b/modules/mesh-delivery-checker/rules/words.go @@ -6,7 +6,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) @@ -183,7 +182,7 @@ func LevelFailure(repository, module, level string, results ...Tested) *meshdeli } for _, f := range res.Failed { name := FailedName(f) - if module == buildrun.NodeBuildModule && f.Package == "" { + if module == nodeBuildModule && f.Package == "" { name = res.Module } all = append(all, FailedFirst{Test: level + ": " + name, Words: FailureWordsAfter(f, HidWords(module, res.By, f))}) From 009c4c6e73be2142a177f599aadf4bb88c077ea0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:32:53 +0100 Subject: [PATCH 7/9] A-97: the seat mesh-builds accepts the build step and says its end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accepted kind step (body StepRun: the pipeline, the branch, the source revision and each entry by module and fingerprint) and the event step-ended. (StepEnded: the same entries, the outcome), as mesh-wants has for its composition step. Builds: design 47 G A-97 (§230, §194, §473) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- contracts/go/seats/meshbuilds/contract.go | 93 +++++++++- .../go/seats/meshbuilds/contract_test.go | 4 + .../seats/mesh-builds/fixtures/events.json | 42 +++++ contracts/seats/mesh-builds/seat.json | 162 ++++++++++++++++++ contracts/ts/seats/meshbuilds/index.d.ts | 69 ++++++++ contracts/ts/seats/meshbuilds/index.js | 15 ++ 6 files changed, 384 insertions(+), 1 deletion(-) diff --git a/contracts/go/seats/meshbuilds/contract.go b/contracts/go/seats/meshbuilds/contract.go index bbf20ee0..836f14e0 100644 --- a/contracts/go/seats/meshbuilds/contract.go +++ b/contracts/go/seats/meshbuilds/contract.go @@ -12,9 +12,14 @@ const ( VerbReserved = "reserved" ) +const ( + AcceptStep = "step" +) + const ( EventPackagePublished = "package-published" EventToolchainAgreed = "toolchain-agreed" + EventStepEnded = "step-ended" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -24,7 +29,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventPackagePublished, EventToolchainAgreed} + return []string{EventPackagePublished, EventToolchainAgreed, EventStepEnded} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -45,6 +50,26 @@ const StreamToolchainAgreed = "SEAT_mesh-builds_toolchain-agreed" // SubjectToolchainAgreed is where toolchain-agreed is published. const SubjectToolchainAgreed = "mesh.seat.mesh-builds.event.toolchain-agreed" +// StreamStepEnded is the stream step-ended is kept in. +const StreamStepEnded = "SEAT_mesh-builds_step-ended" + +// SubjectStepEnded is where step-ended is published, on mesh.seat.mesh-builds.event.step-ended.. +func SubjectStepEnded(node string) string { return "mesh.seat.mesh-builds.event.step-ended." + node } + +// SubjectStepEndedAll hears every step-ended. +const SubjectStepEndedAll = "mesh.seat.mesh-builds.event.step-ended.*" + +// AcceptSubjectStep is where step is sent. +const AcceptSubjectStep = "mesh.seat.mesh-builds.accept.step" + +// BuildEntry is one entry a candidate's build builds: a module at the fingerprint of its build inputs, and the commit that fingerprint was read at. +type BuildEntry struct { + Commit string `json:"commit,omitempty"` + // the module's build-input fingerprint at the commit + Fingerprint string `json:"fingerprint"` + Module string `json:"module"` +} + // Dependency is a dependency the build resolved: one item of kind 1 for each resolved package and one of kind 3 for the toolchain, named by the build module in its step key and carried by the builder (design 47 G 374, 673). type Dependency struct { // the digest of the part, as the build used it @@ -77,6 +102,66 @@ type PackagePublished struct { Version string `json:"version"` } +// StepEnded is the build step's end, said by the builder on the node it runs on (design 47 G A-97; ADR 0666 section 21): the runner takes it only from that holder's account, naming the ask and each entry's fingerprint it was asked at. +type StepEnded struct { + // for a rerun, the caller the runtime proved + By string `json:"by,omitempty"` + Ended *time.Time `json:"ended,omitempty"` + // each entry the step built, by module and fingerprint + Entries []BuildEntry `json:"entries"` + // the ask's id + ID string `json:"id"` + // the node the builder ran it on, as in the subject + Node string `json:"node"` + // never passed for a build that built nothing + Outcome string `json:"outcome"` + Pipeline string `json:"pipeline"` + Rerun bool `json:"rerun,omitempty"` + // why it failed or could not run: the entry not built, the part not heard + Said string `json:"said,omitempty"` + Started *time.Time `json:"started,omitempty"` + Step string `json:"step"` + // the layout version + V int64 `json:"v,omitempty"` +} + +// The values of StepEnded's outcome. +const ( + StepEndedOutcomePassed = "passed" + StepEndedOutcomeFailed = "failed" + StepEndedOutcomeNothingToRun = "nothing to run" + StepEndedOutcomeCouldNotRun = "could not run" +) + +// The values of StepEnded's step. +const ( + StepEndedStepBuild = "build" +) + +// StepRun is the build step of one module's pipeline (design 47 G section 230, A-97): the runner asks the seat mesh-builds to build a candidate's entries, each named by its fingerprint, and the builder, the one entry point for a candidate, takes the ask and answers the step's end. +type StepRun struct { + // the watched branch, or the branch of a run on request + Branch string `json:"branch"` + // for a rerun, the caller the runtime proved + By string `json:"by,omitempty"` + // each entry the candidate's build builds, by module and fingerprint + Entries []BuildEntry `json:"entries"` + // the runner's pipeline key, .., a branch escaped as a key's name is + Pipeline string `json:"pipeline"` + // the step is asked again by the runner's rerun + Rerun bool `json:"rerun,omitempty"` + // the catalog graph revision of the branch the pipeline runs for + Revision int64 `json:"revision,omitempty"` + // the source revision of the watched branch; absent for a run on request + Source int64 `json:"source,omitempty"` + Step string `json:"step"` +} + +// The values of StepRun's step. +const ( + StepRunStepBuild = "build" +) + // ToolchainAgreedDigestsItem is a type of the contract. type ToolchainAgreedDigestsItem struct { // the toolchain part's digest @@ -128,8 +213,14 @@ type ReservedAnswer struct { Version string `json:"version,omitempty"` } +// StepAccept is the body of step. +type StepAccept = StepRun + // PackagePublishedEvent is the body of the event package-published. type PackagePublishedEvent = PackagePublished // ToolchainAgreedEvent is the body of the event toolchain-agreed. type ToolchainAgreedEvent = ToolchainAgreed + +// StepEndedEvent is the body of the event step-ended. +type StepEndedEvent = StepEnded diff --git a/contracts/go/seats/meshbuilds/contract_test.go b/contracts/go/seats/meshbuilds/contract_test.go index 1edf441c..809011ad 100644 --- a/contracts/go/seats/meshbuilds/contract_test.go +++ b/contracts/go/seats/meshbuilds/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v PackagePublishedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.package-published$")}, {name: "a toolchain agreed", subject: "mesh.seat.mesh-builds.event.toolchain-agreed", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"digests\":[{\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"nodes\":[\"ace\",\"g14\"]},{\"digest\":\"sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"nodes\":[\"novox\"]}],\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"digests\":[{\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"nodes\":[\"ace\",\"g14\"]},{\"digest\":\"sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"nodes\":[\"novox\"]}],\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", decode: func(b []byte) (any, error) { var v ToolchainAgreedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.toolchain-agreed$")}, + {name: "the build step ended passed on the builder's node, each entry named by its fingerprint (A-97)", subject: "mesh.seat.mesh-builds.event.step-ended.novox", given: "{\"ended\":\"2026-11-10T10:01:00Z\",\"entries\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\"}],\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"novox\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"build\",\"v\":1}", wire: "{\"ended\":\"2026-11-10T10:01:00Z\",\"entries\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\"}],\"id\":\"m2kx9a1b3c4d6a\",\"node\":\"novox\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"started\":\"2026-11-10T10:00:00Z\",\"step\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "the build step failed on its entry", subject: "mesh.seat.mesh-builds.event.step-ended.novox", given: "{\"entries\":[{\"fingerprint\":\"fp-0123\",\"module\":\"nats\"}],\"id\":\"m2kx9a1b3c4d6b\",\"node\":\"novox\",\"outcome\":\"failed\",\"pipeline\":\"main.12.nats\",\"said\":\"nats:program did not build\",\"step\":\"build\",\"v\":1}", wire: "{\"entries\":[{\"fingerprint\":\"fp-0123\",\"module\":\"nats\"}],\"id\":\"m2kx9a1b3c4d6b\",\"node\":\"novox\",\"outcome\":\"failed\",\"pipeline\":\"main.12.nats\",\"said\":\"nats:program did not build\",\"step\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "reserved asks a module", subject: "", given: "{\"module\":\"mesh-sdk\"}", wire: "{\"module\":\"mesh-sdk\"}", decode: func(b []byte) (any, error) { var v ReservedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "reserved asks a package by its name", subject: "", given: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", wire: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", diff --git a/contracts/seats/mesh-builds/fixtures/events.json b/contracts/seats/mesh-builds/fixtures/events.json index 69f20574..af0d7617 100644 --- a/contracts/seats/mesh-builds/fixtures/events.json +++ b/contracts/seats/mesh-builds/fixtures/events.json @@ -53,5 +53,47 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "the build step ended passed on the builder's node, each entry named by its fingerprint (A-97)", + "of": "event:step-ended", + "subject": "mesh.seat.mesh-builds.event.step-ended.novox", + "given": { + "v": 1, + "id": "m2kx9a1b3c4d6a", + "pipeline": "main.12.nats", + "step": "build", + "node": "novox", + "outcome": "passed", + "entries": [ + { + "module": "nats", + "commit": "0123456789abcdef0123456789abcdef01234567", + "fingerprint": "fp-0123" + } + ], + "started": "2026-11-10T10:00:00Z", + "ended": "2026-11-10T10:01:00Z" + } + }, + { + "name": "the build step failed on its entry", + "of": "event:step-ended", + "subject": "mesh.seat.mesh-builds.event.step-ended.novox", + "given": { + "v": 1, + "id": "m2kx9a1b3c4d6b", + "pipeline": "main.12.nats", + "step": "build", + "node": "novox", + "outcome": "failed", + "entries": [ + { + "module": "nats", + "fingerprint": "fp-0123" + } + ], + "said": "nats:program did not build" + } } ] diff --git a/contracts/seats/mesh-builds/seat.json b/contracts/seats/mesh-builds/seat.json index c984ac2e..b9920457 100644 --- a/contracts/seats/mesh-builds/seat.json +++ b/contracts/seats/mesh-builds/seat.json @@ -78,6 +78,17 @@ } } ], + "accepts": [ + { + "name": "step", + "body": { + "$ref": "#/defs/StepRun" + }, + "callers": [ + "seat:mesh-pipeline-runner" + ] + } + ], "events": [ { "name": "package-published", @@ -98,6 +109,17 @@ "body": { "$ref": "#/defs/ToolchainAgreed" } + }, + { + "name": "step-ended", + "subject": "step-ended.", + "body": { + "$ref": "#/defs/StepEnded" + }, + "keep": { + "max-age-days": 90, + "max-mib": 32 + } } ], "family": { @@ -240,6 +262,146 @@ "digests" ], "description": "the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1)" + }, + "StepRun": { + "type": "object", + "description": "the build step of one module's pipeline (design 47 G section 230, A-97): the runner asks the seat mesh-builds to build a candidate's entries, each named by its fingerprint, and the builder, the one entry point for a candidate, takes the ask and answers the step's end", + "properties": { + "pipeline": { + "type": "string", + "description": "the runner's pipeline key, .., a branch escaped as a key's name is" + }, + "step": { + "type": "string", + "enum": [ + "build" + ] + }, + "branch": { + "type": "string", + "description": "the watched branch, or the branch of a run on request" + }, + "revision": { + "type": "integer", + "description": "the catalog graph revision of the branch the pipeline runs for" + }, + "source": { + "type": "integer", + "description": "the source revision of the watched branch; absent for a run on request" + }, + "rerun": { + "type": "boolean", + "description": "the step is asked again by the runner's rerun" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/BuildEntry" + }, + "description": "each entry the candidate's build builds, by module and fingerprint" + } + }, + "required": [ + "pipeline", + "step", + "branch", + "entries" + ] + }, + "StepEnded": { + "type": "object", + "description": "the build step's end, said by the builder on the node it runs on (design 47 G A-97; ADR 0666 section 21): the runner takes it only from that holder's account, naming the ask and each entry's fingerprint it was asked at", + "properties": { + "v": { + "type": "integer", + "description": "the layout version" + }, + "id": { + "type": "string", + "description": "the ask's id" + }, + "pipeline": { + "type": "string" + }, + "step": { + "type": "string", + "enum": [ + "build" + ] + }, + "node": { + "type": "string", + "description": "the node the builder ran it on, as in the subject" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed", + "nothing to run", + "could not run" + ], + "description": "never passed for a build that built nothing" + }, + "said": { + "type": "string", + "description": "why it failed or could not run: the entry not built, the part not heard" + }, + "rerun": { + "type": "boolean" + }, + "by": { + "type": "string", + "description": "for a rerun, the caller the runtime proved" + }, + "started": { + "type": "string", + "format": "date-time" + }, + "ended": { + "type": "string", + "format": "date-time" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/defs/BuildEntry" + }, + "description": "each entry the step built, by module and fingerprint" + } + }, + "required": [ + "id", + "pipeline", + "step", + "node", + "outcome", + "entries" + ] + }, + "BuildEntry": { + "type": "object", + "description": "one entry a candidate's build builds: a module at the fingerprint of its build inputs, and the commit that fingerprint was read at", + "properties": { + "module": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "fingerprint": { + "type": "string", + "description": "the module's build-input fingerprint at the commit" + } + }, + "required": [ + "module", + "fingerprint" + ] } } } diff --git a/contracts/ts/seats/meshbuilds/index.d.ts b/contracts/ts/seats/meshbuilds/index.d.ts index 426f20d5..d142d6d9 100644 --- a/contracts/ts/seats/meshbuilds/index.d.ts +++ b/contracts/ts/seats/meshbuilds/index.d.ts @@ -8,9 +8,14 @@ export declare const verbs: { readonly reserved: "reserved"; }; +export declare const accepts: { + readonly step: "step"; +}; + export declare const events: { readonly packagePublished: "package-published"; readonly toolchainAgreed: "toolchain-agreed"; + readonly stepEnded: "step-ended"; }; export declare const family: { readonly name: "mesh-builds.events"; readonly maxAgeDays: 7 }; @@ -23,6 +28,22 @@ export declare const toolchainAgreedStream: "SEAT_mesh-builds_toolchain-agreed"; export declare const toolchainAgreedSubject: "mesh.seat.mesh-builds.event.toolchain-agreed"; +export declare const stepEndedStream: "SEAT_mesh-builds_step-ended"; + +export declare function stepEndedSubject(node: string): string; + +export declare const stepEndedSubjectAll: "mesh.seat.mesh-builds.event.step-ended.*"; + +export declare const stepSubject: "mesh.seat.mesh-builds.accept.step"; + +/** one entry a candidate's build builds: a module at the fingerprint of its build inputs, and the commit that fingerprint was read at */ +export interface BuildEntry { + "commit"?: string; + /** the module's build-input fingerprint at the commit */ + "fingerprint": string; + "module": string; +} + /** a dependency the build resolved: one item of kind 1 for each resolved package and one of kind 3 for the toolchain, named by the build module in its step key and carried by the builder (design 47 G 374, 673) */ export interface Dependency { /** the digest of the part, as the build used it */ @@ -55,6 +76,48 @@ export interface PackagePublished { "version": string; } +/** the build step's end, said by the builder on the node it runs on (design 47 G A-97; ADR 0666 section 21): the runner takes it only from that holder's account, naming the ask and each entry's fingerprint it was asked at */ +export interface StepEnded { + /** for a rerun, the caller the runtime proved */ + "by"?: string; + "ended"?: string; + /** each entry the step built, by module and fingerprint */ + "entries": BuildEntry[]; + /** the ask's id */ + "id": string; + /** the node the builder ran it on, as in the subject */ + "node": string; + /** never passed for a build that built nothing */ + "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + "pipeline": string; + "rerun"?: boolean; + /** why it failed or could not run: the entry not built, the part not heard */ + "said"?: string; + "started"?: string; + "step": "build"; + /** the layout version */ + "v"?: number; +} + +/** the build step of one module's pipeline (design 47 G section 230, A-97): the runner asks the seat mesh-builds to build a candidate's entries, each named by its fingerprint, and the builder, the one entry point for a candidate, takes the ask and answers the step's end */ +export interface StepRun { + /** the watched branch, or the branch of a run on request */ + "branch": string; + /** for a rerun, the caller the runtime proved */ + "by"?: string; + /** each entry the candidate's build builds, by module and fingerprint */ + "entries": BuildEntry[]; + /** the runner's pipeline key, .., a branch escaped as a key's name is */ + "pipeline": string; + /** the step is asked again by the runner's rerun */ + "rerun"?: boolean; + /** the catalog graph revision of the branch the pipeline runs for */ + "revision"?: number; + /** the source revision of the watched branch; absent for a run on request */ + "source"?: number; + "step": "build"; +} + export interface ToolchainAgreedDigestsItem { /** the toolchain part's digest */ "digest": string; @@ -104,9 +167,15 @@ export interface ReservedAnswer { "version"?: string; } +/** is the body of step. */ +export type StepAccept = StepRun; + /** is the body of the event package-published. */ export type PackagePublishedEvent = PackagePublished; /** is the body of the event toolchain-agreed. */ export type ToolchainAgreedEvent = ToolchainAgreed; +/** is the body of the event step-ended. */ +export type StepEndedEvent = StepEnded; + diff --git a/contracts/ts/seats/meshbuilds/index.js b/contracts/ts/seats/meshbuilds/index.js index 2a1f4d56..31cc6f7c 100644 --- a/contracts/ts/seats/meshbuilds/index.js +++ b/contracts/ts/seats/meshbuilds/index.js @@ -8,9 +8,14 @@ export const verbs = { reserved: "reserved", }; +export const accepts = { + step: "step", +}; + export const events = { packagePublished: "package-published", toolchainAgreed: "toolchain-agreed", + stepEnded: "step-ended", }; export const family = { name: "mesh-builds.events", maxAgeDays: 7 }; @@ -23,3 +28,13 @@ export const toolchainAgreedStream = "SEAT_mesh-builds_toolchain-agreed"; export const toolchainAgreedSubject = "mesh.seat.mesh-builds.event.toolchain-agreed"; +export const stepEndedStream = "SEAT_mesh-builds_step-ended"; + +export function stepEndedSubject(node) { + return "mesh.seat.mesh-builds.event.step-ended." + node; +} + +export const stepEndedSubjectAll = "mesh.seat.mesh-builds.event.step-ended.*"; + +export const stepSubject = "mesh.seat.mesh-builds.accept.step"; + From 8160dc8d26c796e740aa7753349a6d002bc91b17 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 00:07:53 +0100 Subject: [PATCH 8/9] A-97: the builder takes the runner's build step ask on mesh-builds and answers its end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seat mesh-builds' accepted kind step (cherry-picked contract 0dc8c723 of feat/47-G-recover-runner) names an entry by fingerprint; the builder reads it through the catalog half or the seat mesh-catalog's entry, builds it through its one claim, plan and lanes (a build per fingerprint, no pull request number) and says the end as step-ended. under the ask's id, naming each entry as asked. A build already done is passed at once; the end of a build wakes its waiting step through the log entry. Builds: design 47 G §230, §194, §473, A-84, A-91, A-97, A-99 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../catalogholder_integration_test.go | 2 +- .../cmd/mesh-delivery-builder/main.go | 5 + .../cmd/mesh-delivery-builder/step.go | 101 ++++++++++++ .../step_integration_test.go | 52 ++++++ .../internal/build/run.go | 4 + .../internal/build/step.go | 156 ++++++++++++++++++ .../internal/build/step_test.go | 76 +++++++++ .../mesh-delivery-builder/manifest_test.go | 4 + 8 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/step.go create mode 100644 modules/mesh-delivery-builder/internal/build/step_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go index 9d74cd2e..24600e96 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go @@ -138,7 +138,7 @@ func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[stri } t.Cleanup(func() { _ = s.Unsubscribe() }) for seat, module := range holders { - h.put(t, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", len(seat)%16), 64), ManifestDigest: "sha256:claims-" + module, + h.put(t, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:claims-" + module, Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}}) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index d471a5d4..29797124 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -390,6 +390,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // one taker, so no pattern here covers them). // The seat list over the connection and heard on the hub: a change of it, as of the catalog half's holders, names the build // modules again (design 47 G A-82, §204). + // The runner's build step ask, the builder's one entry point for a candidate (design 47 G A-97). + b.StepEntry = newEntryWaits(branches, half.manifests).entry + if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil { + return err + } seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf) half.holders = func() { seatsChanged(ctx) } for _, pattern := range seatEventPatterns { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go new file mode 100644 index 00000000..87aabd3e --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-builder/internal/build" +) + +// **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat +// mesh-builds' accepted kind `step`, heard on its subject, names the entry of one module by fingerprint; the builder reads it through +// the seat mesh-catalog's `entry`, builds it (build.Builder.BuildStep) and says the end as the seat's event step-ended., the +// ask's id as the event's id, which the runner holds the end to. + +// entryWaits answers an entry by fingerprint from the catalog half or the seat's `entry`, waiting for the answer the entries +// keeper is told of (meshcatalogclient.Entries.Changed), never a timer. +type entryWaits struct { + branches *meshcatalogclient.Branches + manifest *entryManifests + mu sync.Mutex + waiting map[string][]chan meshcatalogclient.EntryRow +} + +func newEntryWaits(branches *meshcatalogclient.Branches, manifests *entryManifests) *entryWaits { + w := &entryWaits{branches: branches, manifest: manifests, waiting: map[string][]chan meshcatalogclient.EntryRow{}} + prior := branches.Entries.Changed + branches.Entries.Changed = func(row meshcatalogclient.EntryRow) { + w.mu.Lock() + chans := w.waiting[row.Fingerprint] + delete(w.waiting, row.Fingerprint) + w.mu.Unlock() + for _, ch := range chans { + ch <- row + } + if prior != nil { + prior(row) + } + } + return w +} + +// entry is build.Builder.StepEntry. +func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { + ch := make(chan meshcatalogclient.EntryRow, 1) + w.mu.Lock() + w.waiting[fingerprint] = append(w.waiting[fingerprint], ch) + w.mu.Unlock() + e, state := w.branches.Resolve(ctx, fingerprint) + if state == meshcatalogclient.EntryAsked { + select { + case row := <-ch: + if row.Entry == nil { + return meshcatalog.CatalogEntry{}, fmt.Errorf("the seat mesh-catalog holds no entry of it (%s)", row.State) + } + e, state = *row.Entry, row.State + case <-ctx.Done(): + return meshcatalog.CatalogEntry{}, ctx.Err() + } + } + if state != meshcatalogclient.EntryAnswered { + return meshcatalog.CatalogEntry{}, fmt.Errorf("the seat mesh-catalog's entry of it is %s", state) + } + return e, w.manifest.keep(e) // the manifest the build is planned from +} + +// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine. +func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error { + sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) { + id := m.Header.Get("x-event-id") + if id == "" { + id = m.Header.Get(nats.MsgIdHdr) + } + var run meshbuilds.StepRun + if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild { + logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err) + return + } + go func() { + end := b.BuildStep(ctx, run) + end.ID = id + if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil { + logf("saying the end of the build step %s: %v", id, err) + } + }() + }) + if err != nil { + return err + } + go func() { <-ctx.Done(); _ = sub.Drain() }() + return nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go new file mode 100644 index 00000000..8041973f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the +// module's one claim and answered as the seat's step-ended. under the ask's id (design 47 G A-97, A-99). +func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) { + b := startBed(t) + b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") + sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) + if err != nil { + t.Fatal(err) + } + defer sub.Close() + ended := make(chan *nats.Msg, 4) + s, err := sub.ChanSubscribe(meshbuilds.SubjectStepEndedAll, ended) + if err != nil { + t.Fatal(err) + } + defer s.Unsubscribe() + b.start(t, "n1", nil) + b.await(t, "netcheck", isState(builds.BuildDone)) + run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1, + Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}} + data, _ := json.Marshal(run) + msg := nats.NewMsg(meshbuilds.AcceptSubjectStep) + msg.Data = data + msg.Header.Set("x-event-id", "ask-1") + if err := sub.PublishMsg(msg); err != nil { + t.Fatal(err) + } + select { + case m := <-ended: + var end meshbuilds.StepEnded + if err := json.Unmarshal(m.Data, &end); err != nil || end.ID != "ask-1" || end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Pipeline != "main.1.netcheck" || m.Subject != meshbuilds.SubjectStepEnded("n1") { + t.Fatalf("step-ended on %s = %+v, %v; want the ask ask-1 passed on n1", m.Subject, end, err) + } + case <-time.After(20 * time.Second): + t.Fatal("no step-ended within 20 s") + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index b10dcfac..6188ae0f 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -41,6 +41,8 @@ type ToolchainEntries interface { // change of the toolchains. type Builder struct { Claimer Claimer + // StepEntry reads the entry of a fingerprint through the seat mesh-catalog's entry (design 47 G A-84, A-97). + StepEntry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) // Seats finds the build modules: the holders of the build seats on the seat list and the catalog half (design 47 G A-82). Seats *buildrun.BuildSeats // Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq @@ -129,6 +131,7 @@ type Builder struct { inflight map[string]bool // by build key again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built plans map[string]planned // by manifest digest + stepWaits map[string][]chan builds.BuildLogEntry // by build key: the step asks waiting for its end (step.go) cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go) via map[string]via // by build key: a candidate's fetch (candidates.go) // outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them. @@ -1202,6 +1205,7 @@ func (b *Builder) now() time.Time { // logBuild tells the log an entry, said in the journal when it cannot be one. func (b *Builder) logBuild(e builds.BuildLogEntry) { + b.stepEnded(e) if b.Log == nil { return } diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go new file mode 100644 index 00000000..506d12ac --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -0,0 +1,156 @@ +package build + +import ( + "context" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// **A candidate's build is the runner's step ask** (design 47 G A-97, A-84, A-99): the runner's asking half asks the seat +// mesh-builds a `build` step naming each entry by fingerprint, and the builder, its one entry point for a candidate, reads each +// entry through the seat mesh-catalog's `entry` (the repository and the commit are the member's head), builds it as it builds any +// fingerprint (the existing claim, plan and lanes) and answers the step's end. A build is per fingerprint, so no pull request +// number is needed, and the builder follows no candidate. + +// BuildStep builds the entries of one step ask and answers its end: passed when every entry's build is done, failed when one +// failed (said), could not run when an entry could not be read, nothing to run for an ask naming none. The ask's id and the node +// are the caller's to set. +func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbuilds.StepEnded { + b.init() + started := b.now() + end := meshbuilds.StepEnded{V: 1, Pipeline: run.Pipeline, Step: meshbuilds.StepEndedStepBuild, Rerun: run.Rerun, By: run.By, Entries: run.Entries, Node: b.Node, Started: &started} + finish := func(outcome, said string) meshbuilds.StepEnded { + at := b.now() + end.Ended, end.Outcome, end.Said = &at, outcome, said + return end + } + if len(run.Entries) == 0 { + return finish(meshbuilds.StepEndedOutcomeNothingToRun, "the ask names no entry") + } + if b.StepEntry == nil { + return finish(meshbuilds.StepEndedOutcomeCouldNotRun, "the builder reads no entry of the seat mesh-catalog") + } + var failed []string + for _, e := range run.Entries { + outcome, said := b.buildEntry(ctx, e) + switch outcome { + case meshbuilds.StepEndedOutcomeCouldNotRun: + return finish(outcome, said) + case meshbuilds.StepEndedOutcomeFailed: + failed = append(failed, said) + } + } + if len(failed) > 0 { + return finish(meshbuilds.StepEndedOutcomeFailed, strings.Join(failed, "; ")) + } + return finish(meshbuilds.StepEndedOutcomePassed, "") +} + +// buildEntry builds one entry at its fingerprint and waits for its end. +func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { + entry, err := b.StepEntry(ctx, e.Fingerprint) + if err != nil { + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err) + } + if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) { + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit) + } + key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) + if err != nil { + return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() + } + ended := b.awaitEnd(key) + defer b.dropEnd(key, ended) + if done, failedWords, found := b.keyEnd(ctx, key); found { + if done { + return meshbuilds.StepEndedOutcomePassed, "" + } + return meshbuilds.StepEndedOutcomeFailed, failedWords + } + m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path, + Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages} + b.mu.Lock() + if _, onTrunk := b.trunkFP[key]; !onTrunk { + b.via[key] = via{} + } + b.mu.Unlock() + b.consider(ctx, entry.Module, m) + select { + case log := <-ended: + if log.Outcome == builds.BuildDone { + return meshbuilds.StepEndedOutcomePassed, "" + } + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error) + case <-ctx.Done(): + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the builder stopped before its build ended", entry.Module, entry.Fp) + } +} + +// keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended. +func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) { + raw, _, ok, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !ok { + return false, "", false + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil { + return false, "", false + } + switch { + case v.State == builds.BuildDone && !Unpublished(v, b.Seats): + return true, "", true + case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error): + return false, key + ": " + v.Error, true + } + return false, "", false +} + +// awaitEnd registers a wait for the end of the build of key, heard at its log entry (logBuild). +func (b *Builder) awaitEnd(key string) chan builds.BuildLogEntry { + ch := make(chan builds.BuildLogEntry, 1) + b.mu.Lock() + if b.stepWaits == nil { + b.stepWaits = map[string][]chan builds.BuildLogEntry{} + } + b.stepWaits[key] = append(b.stepWaits[key], ch) + b.mu.Unlock() + return ch +} + +func (b *Builder) dropEnd(key string, ch chan builds.BuildLogEntry) { + b.mu.Lock() + defer b.mu.Unlock() + for i, c := range b.stepWaits[key] { + if c == ch { + b.stepWaits[key] = append(b.stepWaits[key][:i], b.stepWaits[key][i+1:]...) + break + } + } + if len(b.stepWaits[key]) == 0 { + delete(b.stepWaits, key) + } +} + +// stepEnded tells the step asks waiting for a build that its end was logged. +func (b *Builder) stepEnded(e builds.BuildLogEntry) { + if e.Event != builds.BuildEndedEvent { + return + } + key, err := builds.BuilderBuildKey(e.Module, e.FP) + if err != nil { + return + } + b.mu.Lock() + chans := append([]chan builds.BuildLogEntry(nil), b.stepWaits[key]...) + b.mu.Unlock() + for _, ch := range chans { + select { + case ch <- e: + default: + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go new file mode 100644 index 00000000..6cadbc28 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -0,0 +1,76 @@ +package build + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The runner's build step ask is the builder's one entry point for a candidate (design 47 G A-97): each entry is read by its +// fingerprint, built through the one claim, and the step's end says passed only for a build that is done. +func TestABuildStepAnswersItsEnd(t *testing.T) { + r := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"}) + entryOf := func(fp, commit string) meshcatalog.CatalogEntry { + return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: commit, Fp: fp, ManifestDigest: mdigest('a'), Path: "."} + } + r.b.StepEntry = func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) { + if fp == fp1 { + return entryOf(fp1, "c1"), nil + } + return meshcatalog.CatalogEntry{}, errors.New("not found") + } + ask := func(entries ...meshbuilds.BuildEntry) meshbuilds.StepEnded { + return r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries}) + } + if end := ask(); end.Outcome != meshbuilds.StepEndedOutcomeNothingToRun { + t.Errorf("an ask naming no entry: %+v", end) + } + if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: "src2:" + strings.Repeat("2", 64)}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "cannot be read") { + t.Errorf("an entry the seat does not hold: %+v", end) + } + if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: fp1, Commit: "other"}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun { + t.Errorf("an entry at another commit: %+v", end) + } + // A build already done is passed at once, naming each entry as it was asked. + done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a')}) + if _, err := r.st.Create(t.Context(), r.key, done); err != nil { + t.Fatal(err) + } + entries := []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1, Commit: "c1"}} + if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" { + t.Errorf("a done build: %+v", end) + } +} + +// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97). +func TestAStepAskIsWokenByItsBuildsEnd(t *testing.T) { + r := newRig(t) + key := r.key + ch := r.b.awaitEnd(key) + r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildStartedEvent, Module: "m", FP: fp1}) // a start wakes nothing + select { + case <-ch: + t.Fatal("a build's start woke the wait") + default: + } + r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: fp1, Outcome: builds.BuildDone}) + select { + case e := <-ch: + if e.Outcome != builds.BuildDone { + t.Errorf("end = %+v", e) + } + case <-time.After(time.Second): + t.Fatal("the end of the build did not wake the wait") + } + r.b.dropEnd(key, ch) + if len(r.b.stepWaits) != 0 { + t.Errorf("waits left: %v", r.b.stepWaits) + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 6914f076..807e7a61 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -260,6 +260,10 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR return err } for _, f := range forbidden { + // The one subscription is the seat mesh-builds' accepted kind `step`, the runner's build ask (design 47 G A-97). + if f == "Subscribe(" && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/step.go" { + continue + } if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the builder serves no request subject and holds no timer", path, f) } From a2dfbec30da3dac902dd9237dd4f228b2c761af8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 00:10:29 +0100 Subject: [PATCH 9/9] The builder no longer watches the gate's source for candidate base merges A candidate comes by the runner's build step ask (A-97), so the builder's trunkSource watch of the gate's source, its sourceHeads, the wake it gave a floor, its read mesh-delivery-gate.source and the bed's source bucket go. Builds: design 47 G A-97, A-84 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../catalogholder_integration_test.go | 17 +++-- .../mesh-delivery-builder/integration_test.go | 20 +----- .../cmd/mesh-delivery-builder/main.go | 63 ----------------- .../cmd/mesh-delivery-builder/names.go | 6 -- .../cmd/mesh-delivery-builder/names_test.go | 3 +- .../cmd/mesh-delivery-builder/optional.go | 68 ------------------- .../cmd/mesh-delivery-builder/refused_test.go | 12 ---- .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 3 +- 9 files changed, 17 insertions(+), 177 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go index 24600e96..699e8cb4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go @@ -86,12 +86,19 @@ func (h *catalogHolder) whole() meshcatalog.RevisionAnswer { } // put sets one module's entry in a new trunk revision of main and says catalog-revision. -func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { +func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { h.putAll(t, e) } + +// putAll sets several modules' entries in one new trunk revision of main. +func (h *catalogHolder) putAll(t *testing.T, es ...meshcatalog.CatalogEntry) { t.Helper() h.mu.Lock() h.revision++ - h.entries[e.Module] = e - ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: []string{e.Module}, V: 1}) + var changed []string + for _, e := range es { + h.entries[e.Module] = e + changed = append(changed, e.Module) + } + ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: changed, V: 1}) h.mu.Unlock() if _, err := h.js.Publish(h.ctx, meshcatalog.SubjectCatalogRevision("main"), ev); err != nil { t.Fatal(err) @@ -137,8 +144,10 @@ func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[stri t.Fatal(err) } t.Cleanup(func() { _ = s.Unsubscribe() }) + var entries []meshcatalog.CatalogEntry for seat, module := range holders { - h.put(t, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:claims-" + module, + entries = append(entries, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:" + fmt.Sprintf("%064x", len(module)*7919+int(module[len(module)-1])), Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}}) } + h.putAll(t, entries...) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index d53136cf..cdbc6a29 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -28,15 +28,12 @@ import ( runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" - "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views/viewstest" @@ -201,23 +198,9 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { } } -// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit. +// trunkHead is a trunk head of novox/mesh-catalog. const trunkHead = "3333333333333333333333333333333333333333" -// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision. -func (b *bed) trunk(t *testing.T, head string) uint64 { - t.Helper() - data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{V: keys.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}}) - if err != nil { - t.Fatal(err) - } - rev, err := b.source.Put(b.ctx, meshdeliveryclient.SourceKey("main"), data) - if err != nil { - t.Fatal(err) - } - return rev -} - type bed struct { ctx context.Context js jetstream.JetStream @@ -298,7 +281,6 @@ func startBedWith(t *testing.T, closed bool) *bed { } return kv } - b.source = mk(sourceBucket, sourceHistory) // The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's // verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2 // each have their durable consumer. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 29797124..3edd0cb5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -10,13 +10,10 @@ import ( "fmt" "io" "log" - "maps" "os" "os/signal" - "slices" "strconv" "strings" - "sync" "syscall" "time" @@ -31,12 +28,10 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/sdk/buildstep" - "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" @@ -236,18 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return fmt.Sprintf("built again as revision %d", r.Revision), false }} - // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base - // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision - // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it - // cannot be read the trunk's keys are built and the candidates wait (issue 1945). errs := make(chan error, 10) - trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) { - // A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*). - (&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "") - b.TrunkMoved(ctx) - }, errs: errs, logf: logger.Printf} - trunks.start() - b.Trunk = trunks.head b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf} b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the @@ -505,50 +489,3 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder's own state of its first views (views.StatusState); its findings (findingsBucket) are where the condition // keeper raises a first view unanswered past its bound (ADR 0615 §6). var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState - -// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest -// sourceHistory of them: what a candidate's base revision names. -type sourceHeads struct { - mu sync.Mutex - byRev map[uint64]map[string]string -} - -// put takes one revision in and answers whether it is new. -func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool { - if !opOf(e) { - return false - } - src, err := meshdeliveryclient.DecodeSource(e.Value()) - if err != nil { - return false - } - s.mu.Lock() - defer s.mu.Unlock() - if s.byRev == nil { - s.byRev = map[uint64]map[string]string{} - } - if _, had := s.byRev[e.Revision()]; had { - return false - } - s.byRev[e.Revision()] = src.Heads - for len(s.byRev) > sourceHistory { - delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev)))) - } - return true -} - -// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more). -func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) { - s.mu.Lock() - defer s.mu.Unlock() - heads, ok := s.byRev[rev] - if !ok { - return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev) - } - for r, head := range heads { - if trunk.SameRepository(r, repository) { - return head, nil - } - } - return "", nil -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go index 53c8b837..1c4b475a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go @@ -18,12 +18,6 @@ const ( indexerModule = "mesh-delivery-indexer" // assignerModule is the holder of the state assignments. assignerModule = buildrun.AssignerModule - - // sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names. - sourceBucket = gateModule + "_source" - - // The revisions the gate's source keeps per key (the bed of the integration test creates it alike). - sourceHistory = 64 ) // namedTreePrefix opens the tree input of a module whose every artifact names its paths. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go index 00b38e80..0e646bd0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go @@ -26,7 +26,6 @@ func TestTheNamesAreTheManifests(t *testing.T) { t.Fatal(err) } for _, read := range []string{ - gateModule + ".source", "node-build-go.steps", "node-build-image.steps", "node-build-node.steps", } { if !slices.Contains(m.Reads, read) { @@ -45,7 +44,7 @@ func TestTheNamesAreTheManifests(t *testing.T) { } } for want, got := range map[string]string{ - "mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), + "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"), buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket, } { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go index 87ba0c67..7bfb1320 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go @@ -1,76 +1,8 @@ package main -import ( - "context" - "fmt" - "sync" - - "github.com/nats-io/nats.go/jetstream" -) - // **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of // mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant // reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet: // exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds, // the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left, // and opened again at the next event that needs it where there is one. - -// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and, -// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys, -// and a candidate's preview waits, saying why. -type trunkSource struct { - ctx context.Context - js bucketOpener - heads *sourceHeads - moved func(context.Context) - errs chan<- error - logf func(string, ...any) - - mu sync.Mutex - opened bool -} - -// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask. -func (t *trunkSource) ensure() error { - t.mu.Lock() - defer t.mu.Unlock() - if t.opened { - return nil - } - kv, err := t.js.KeyValue(t.ctx, sourceBucket) - if err != nil { - return fmt.Errorf("the bucket %s cannot be read (%v)", sourceBucket, err) - } - w, err := kv.WatchAll(t.ctx, jetstream.IncludeHistory()) // every watched branch's key: a revision names its head whichever branch it is of (design 47 G §15) - if err != nil { - return fmt.Errorf("the bucket %s cannot be watched (%v)", sourceBucket, err) - } - sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) }) - if err != nil { - return fmt.Errorf("the bucket %s: %w", sourceBucket, err) - } - t.opened = true - go func() { - t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) { - if t.heads.put(e) && t.moved != nil { - t.moved(t.ctx) - } - }) - }() - return nil -} - -// start opens the source at start; one that cannot be is said, and the builder goes on. -func (t *trunkSource) start() { - if err := t.ensure(); err != nil { - t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err) - } -} - -// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet. -func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) { - if err := t.ensure(); err != nil { - return "", err - } - return t.heads.head(ctx, rev, repository) -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index dee20974..34d6449c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -43,23 +43,11 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard) t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding) t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard) - var logs bytes.Buffer - logger := log.New(&logs, "", 0) refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")} if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) } - // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk - // head is told why and opens it again then. - src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} - src.start() - if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), sourceBucket) { - t.Errorf("a trunk head with the source refused: %v, want why", err) - } - if src.opened { - t.Error("a refused source counts as opened, so it is never opened again") - } t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads) } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 807e7a61..950de63c 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -118,7 +118,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a // node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3). - if want := []string{"mesh-delivery-assigner.assignments", "mesh-delivery-gate.source", + if want := []string{"mesh-delivery-assigner.assignments", "node-build-go.steps", "node-build-image.steps", "node-build-node.steps"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 49972d01..0336fd6f 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -142,8 +142,7 @@ "node-build-go.steps", "node-build-image.steps", "node-build-node.steps", - "mesh-delivery-assigner.assignments", - "mesh-delivery-gate.source" + "mesh-delivery-assigner.assignments" ], "uses": [ "node-build-go",