From 88a9d19b4fa33a71a3a2ce99d12f8ed921375757 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:51:27 +0100 Subject: [PATCH] DLV-B2: a package is numbered under its name in its ecosystem, read from the indexer's packages state The builder numbers, asks the floor of, records and tags a package under the name the indexer's series key names for its / (a Go module path, an npm package name; design 32), so builder_versions answers the holder's question by the name a registry seat publishes it under. A package the state names nothing for, or a state not readable, is built unnumbered and said; the rest of the claim is numbered. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-builder/main.go | 3 + .../cmd/mesh-delivery-builder/tagbounds.go | 2 +- .../cmd/mesh-delivery-builder/versions.go | 34 +++++++ .../internal/versions/claim.go | 45 ++++++++- .../internal/versions/names.go | 52 ++++++++++ .../internal/versions/tagbound.go | 7 +- .../internal/versions/versions_test.go | 98 ++++++++++++++++++- .../mesh-delivery-builder/manifest_test.go | 3 +- modules/mesh-delivery-builder/module.json | 3 +- 9 files changed, 232 insertions(+), 15 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/versions/names.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index d2d3a8f5..6834c432 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -129,7 +129,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Ready(func(kv jetstream.KeyValue) { floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf)) }) + // A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32). + packagesKV := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is numbered: each is built unnumbered and said", logger.Printf) numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, + Names: versions.NamesIn(indexerPackages{packagesKV}), Said: func(nb versions.NumberedBody) { if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil { logger.Printf("%v", err) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go index 8a2454b4..4c6d3618 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/tagbounds.go @@ -110,7 +110,7 @@ func tagNumbered(kv func() jetstream.KeyValue, logf func(string, ...any), now fu if err != nil { break } - t, changed := t.WithNumbered(c.Module, c.Head, got.Versions, now()) + t, changed := t.WithNumbered(c.Module, c.Head, got.Versions, got.Names, now()) if !changed { break } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index 1726c4e3..feabc77b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -53,6 +53,40 @@ func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, b return e.Value(), e.Revision(), true, nil } +// indexerPackages is the indexer's packages state (delivery.PackagesBucket, the manifest's reads grant) as the builder +// reads a package's name in its ecosystem from it (design 32): bound once the bus holds it; until then a package is +// built unnumbered and said (versions.Numberer.Names). +type indexerPackages struct{ b *process.AwaitedBucket } + +var errPackagesMissing = errors.New("the indexer's state " + delivery.PackagesBucket + " is not on the bus yet") + +func (s indexerPackages) Keys(ctx context.Context) ([]string, error) { + kv := s.b.KV() + if kv == nil { + return nil, errPackagesMissing + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func (s indexerPackages) Get(ctx context.Context, key string) ([]byte, bool, error) { + kv := s.b.KV() + if kv == nil { + return nil, false, errPackagesMissing + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + func versionsConflict(err error) error { if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { return versions.ErrConflict diff --git a/modules/mesh-delivery-builder/internal/versions/claim.go b/modules/mesh-delivery-builder/internal/versions/claim.go index 08f0791a..957ec6b1 100644 --- a/modules/mesh-delivery-builder/internal/versions/claim.go +++ b/modules/mesh-delivery-builder/internal/versions/claim.go @@ -35,10 +35,16 @@ func SeatOf(kind, language string) (seat, numberedAs string, ok bool) { return "", "", false } -// NameOf is the name a module's artifact is numbered under: `/`, the name the seat answers it by -// (ADR 0605 §9's `//@…`). +// NameOf is the name a module's image, bundle or archive is numbered under: `/`, the name the seat +// answers it by (ADR 0605 §9's `//@…`). A package is numbered under its name in its own +// ecosystem instead (Numberer.Names). func NameOf(module, artifact string) string { return module + "/" + artifact } +// PackageNames answers a package's name in its own ecosystem (design 32: a Go package's module path, an npm package's +// package.json name) for the module and artifact it is built from, as the indexer's packages state holds it; known is +// false when the state names none for it. An error is a state that cannot be read. +type PackageNames func(ctx context.Context, module, artifact string) (name string, known bool, err error) + // TagPrefix is the git tag prefix of a package in the module's directory path: `go/v` for mesh-sdk's `go`, `v` at the // root (design 51: the builder tags each Go version `go/v0.1.`). func TagPrefix(path string) string { @@ -76,6 +82,17 @@ type Numbering struct { Wait string // Said are the builder's narrations of each decision, for its process log and its build log (build.said). Said []Narration + // Names are the names each numbered artifact was numbered under, by artifact (NameOf, or a package's name in its + // ecosystem). + Names map[string]string +} + +// NameOf is the name artifact of module was numbered under in this numbering. +func (g Numbering) NameOf(module, artifact string) string { + if name := g.Names[artifact]; name != "" { + return name + } + return NameOf(module, artifact) } // Narration is one line the builder says in its build log (design 51: the build log says a version was kept, and @@ -117,13 +134,15 @@ type Numberer struct { Holder func(seat string) string // Said is told each version newly numbered; nil tells nobody. Said func(NumberedBody) - Now func() time.Time + // Names answers a package's name in its ecosystem; nil numbers a package as NameOf names it. + Names PackageNames + Now func() time.Time } // Number numbers every numbered artifact of c. An error is a store that cannot be read or written: the claim is then // built as before, unnumbered, and said. func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) { - out := Numbering{Versions: map[string]string{}} + out := Numbering{Versions: map[string]string{}, Names: map[string]string{}} arts := append([]ClaimArtifact(nil), c.Artifacts...) sort.Slice(arts, func(i, j int) bool { return arts[i].Artifact < arts[j].Artifact }) for _, a := range arts { @@ -132,6 +151,22 @@ func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) continue } name := NameOf(c.Module, a.Artifact) + if kind == KindPackage && n.Names != nil { + // A package is numbered, asked for its floor and published under its name in its ecosystem (design 32), the + // name the holder's builder_versions question and the seat's versions name it by. + named, known, err := n.Names(ctx, c.Module, a.Artifact) + if err != nil || !known { + why := "the indexer's packages state names no name of it in its ecosystem yet (it writes one at the module's next trunk merge)" + if err != nil { + why = "the indexer's packages state cannot be read: " + err.Error() + } + out.Unnumbered = append(out.Unnumbered, a.Artifact) + out.Said = append(out.Said, Narration{Line: name + ": built unnumbered: " + why, Why: SaidUnnumbered}) + continue + } + name = named + } + out.Names[a.Artifact] = name entries, err := n.Versions.Of(ctx, name) if err != nil { return out, err @@ -196,7 +231,7 @@ func (n *Numberer) Built(ctx context.Context, c BuildClaim, got Numbering, diges if d == "" { continue } - if _, err := n.Versions.Mark(ctx, NameOf(c.Module, artifact), version, func(e *Entry) { + if _, err := n.Versions.Mark(ctx, got.NameOf(c.Module, artifact), version, func(e *Entry) { // The new digest replaces the recorded one only while no published was said for the number (design 51 // *Tools of the builder*): a rebuild of a gone file under the same number. if e.Digest == "" || e.State == Numbered || e.State == Refused { diff --git a/modules/mesh-delivery-builder/internal/versions/names.go b/modules/mesh-delivery-builder/internal/versions/names.go new file mode 100644 index 00000000..65b57eff --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/names.go @@ -0,0 +1,52 @@ +package versions + +import ( + "context" + "encoding/json" + "sort" + "strings" +) + +// seriesName is what the builder reads of a series key of the indexer's packages state (delivery.PackagesState): the +// package's name in its ecosystem and the `/` it is built from. The value's type is the indexer's own +// (its SeriesPackage, fields `name` and `package`); the fields are read here by their names, not by its type. +type seriesName struct { + Name string `json:"name"` + Package string `json:"package"` +} + +// PackagesState is the indexer's packages state as the builder reads it (its manifest's reads grant): its keys, and one +// key's value (found false when it holds none). +type PackagesState interface { + Keys(ctx context.Context) ([]string, error) + Get(ctx context.Context, key string) (value []byte, found bool, err error) +} + +// NamesIn are a package's names in its ecosystem as the indexer's packages state holds them (design 32): the name of the +// series key that names module and artifact as what it is built from. A series key is named `.`; a key +// with no dot is a mesh Go module's published packages, not a series. A value that does not decode is passed over. +func NamesIn(s PackagesState) PackageNames { + return func(ctx context.Context, module, artifact string) (string, bool, error) { + keys, err := s.Keys(ctx) + if err != nil { + return "", false, err + } + sort.Strings(keys) + built := NameOf(module, artifact) + for _, key := range keys { + if !strings.Contains(key, ".") { + continue + } + value, found, err := s.Get(ctx, key) + if err != nil { + return "", false, err + } + var v seriesName + if !found || json.Unmarshal(value, &v) != nil || v.Package != built || v.Name == "" { + continue + } + return v.Name, true, nil + } + return "", false, nil + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/tagbound.go b/modules/mesh-delivery-builder/internal/versions/tagbound.go index b8e4b861..51cfdc83 100644 --- a/modules/mesh-delivery-builder/internal/versions/tagbound.go +++ b/modules/mesh-delivery-builder/internal/versions/tagbound.go @@ -23,8 +23,9 @@ func (t TagAsk) Unanswered() bool { } // WithNumbered is the tag ask once a check's build of its head numbered module's packages under its tag: numbered, each -// version by its name, the answer naming them. An untag, or an ask of another head, is left as it is. -func (t TagAsk) WithNumbered(module, head string, got map[string]string, at time.Time) (TagAsk, bool) { +// version by the name it was numbered under (names, by artifact; NameOf where it names none), the answer naming them. An +// untag, or an ask of another head, is left as it is. +func (t TagAsk) WithNumbered(module, head string, got, names map[string]string, at time.Time) (TagAsk, bool) { if t.Untag || t.Head != head || len(got) == 0 || t.State == TagRemoving || t.State == TagRemoved { return t, false } @@ -33,7 +34,7 @@ func (t TagAsk) WithNumbered(module, head string, got map[string]string, at time } changed := false for artifact, v := range got { - name := NameOf(module, artifact) + name := Numbering{Names: names}.NameOf(module, artifact) if t.Versions[name] != v { t.Versions[name], changed = v, true } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index f138f16e..22eaa716 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "errors" + "maps" "reflect" + "slices" "strings" "sync" "testing" @@ -88,6 +90,23 @@ func (k keeperMem) keys() ([]string, error) { } func (k keeperMem) get(key string) (json.RawMessage, bool, error) { v, ok := k[key]; return v, ok, nil } +// packagesMem is the indexer's packages state as the builder reads it, in memory. +type packagesMem map[string][]byte + +func (p packagesMem) Keys(context.Context) ([]string, error) { + if p == nil { + return nil, errors.New("not readable") + } + return slices.Collect(maps.Keys(p)), nil +} + +func (p packagesMem) Get(_ context.Context, key string) ([]byte, bool, error) { + v, ok := p[key] + return v, ok, nil +} + +func (p packagesMem) failing() packagesMem { return nil } + func known(seat, tags, reqs []string) Floor { return Floor{Seat: seat, Tags: tags, Requirements: reqs} } // TestTheBuildersVersions holds design 51 *The versions*, *The floor* and *Tagged versions* (DLV-B1, DLV-B2, DLV-B3): @@ -450,6 +469,77 @@ func TestTheBuildersVersions(t *testing.T) { t.Fatal("a published of a version the builder never numbered is not said") } }) + t.Run("a package is numbered under its name in its ecosystem", func(t *testing.T) { + // design 32: the indexer's packages state names, in each series key, the package's name and the / + // it is built from; the builder numbers, asks the floor of and records the package under that name. + ctx := context.Background() + state := packagesMem{ + "git=2enovox=2ebe/novox/mesh-sdk/go.0.1": []byte(`{"v":1,"name":"git.novox.be/novox/mesh-sdk/go","package":"mesh-sdk/go","series":"0.1"}`), + "@novox/mesh-sdk.0.1": []byte(`{"v":1,"name":"@novox/mesh-sdk","package":"mesh-sdk/ts","series":"0.1"}`), + "git=2enovox=2ebe/novox/mesh-sdk/go": []byte(`{"v":1,"module":"git.novox.be/novox/mesh-sdk/go"}`), + "broken.0.1": []byte(`not json`), + } + var floorOf []string + src := FloorSources{ + Seat: func(_ context.Context, _, name string) ([]string, error) { + floorOf = append(floorOf, name) + return []string{"0.1.3"}, nil + }, + Tags: func(context.Context, string, string) ([]string, error) { return nil, nil }, + Requirements: func(_ context.Context, name string) ([]string, error) { + floorOf = append(floorOf, name) + return nil, nil + }, + } + var said []NumberedBody + n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Names: NamesIn(state), Said: func(b NumberedBody) { said = append(said, b) }} + c := BuildClaim{Module: "mesh-sdk", Path: "go", Repository: "novox/mesh-sdk", Fingerprint: "fp1", Commit: "c1", Revision: "3", Merge: 3, + Artifacts: []ClaimArtifact{{Artifact: "go", Kind: KindPackage, Language: "go"}, {Artifact: "ts", Kind: KindPackage, Language: "typescript"}, + {Artifact: "new", Kind: KindPackage, Language: "go"}, {Artifact: "program", Kind: KindBundle, Language: "go"}}} + got, err := n.Number(ctx, c) + if err != nil || got.Versions["go"] != "0.1.4" || got.Versions["ts"] != "0.1.4" || !reflect.DeepEqual(got.Unnumbered, []string{"new"}) { + t.Fatalf("Number = %+v, %v; want go and ts numbered, new unnumbered (no name yet)", got, err) + } + if got.Names["go"] != "git.novox.be/novox/mesh-sdk/go" || got.Names["ts"] != "@novox/mesh-sdk" || got.Names["program"] != "mesh-sdk/program" { + t.Errorf("names = %v", got.Names) + } + if !slices.Contains(floorOf, "git.novox.be/novox/mesh-sdk/go") || slices.Contains(floorOf, "mesh-sdk/go") { + t.Errorf("the floor was asked of %v, want the Go module path", floorOf) + } + if es, _ := n.Versions.Of(ctx, "git.novox.be/novox/mesh-sdk/go"); len(es) != 1 || es[0].Version != "0.1.4" { + t.Errorf("versions of the Go module path = %+v", es) + } + if es, _ := n.Versions.Of(ctx, "mesh-sdk/go"); len(es) != 0 { + t.Errorf("numbered under the builder's name too: %+v", es) + } + names := map[string]bool{} + for _, b := range said { + names[b.Name] = true + } + if !names["git.novox.be/novox/mesh-sdk/go"] || !names["@novox/mesh-sdk"] { + t.Errorf("version.numbered said %v, want the ecosystem names", names) + } + if err := n.Built(ctx, c, got, map[string]string{"go": "sha256:aa"}); err != nil { + t.Fatal(err) + } + if es, _ := n.Versions.Of(ctx, "git.novox.be/novox/mesh-sdk/go"); len(es) != 1 || es[0].Digest != "sha256:aa" { + t.Errorf("the built digest is not recorded under the name: %+v", es) + } + tagged, _ := TagAsk{Head: "h", State: TagAsked}.WithNumbered("mesh-sdk", "h", map[string]string{"go": "0.1.4-x.1"}, got.Names, time.Unix(1, 0)) + if tagged.Versions["git.novox.be/novox/mesh-sdk/go"] != "0.1.4-x.1" { + t.Errorf("a tag ask names %v, want the Go module path", tagged.Versions) + } + if _, _, err := NamesIn(packagesMem(nil).failing())(ctx, "m", "a"); err == nil { + t.Error("a packages state that cannot be read is no error") + } + // A packages state not on the bus yet leaves the package unnumbered, said, and numbers the rest of the claim. + n.Names = NamesIn(packagesMem(nil).failing()) + bare, err := n.Number(ctx, c) + if err != nil || !reflect.DeepEqual(bare.Unnumbered, []string{"go", "new", "ts"}) || bare.Versions["program"] == "" || + !strings.Contains(bare.Said[0].Line, "cannot be read") { + t.Errorf("an unreadable packages state: %+v, %v", bare, err) + } + }) t.Run("fill", func(t *testing.T) { if _, full := Fill(Cap*79/100, Cap); full { t.Error("79 % is full") @@ -645,17 +735,17 @@ func TestTheBuildersVersions(t *testing.T) { if !a.Unanswered() { t.Errorf("a kept ask with no version is answered: %+v", a) } - if _, changed := a.WithNumbered("mesh-sdk", "other", map[string]string{"go": "0.1.15-feat-x-y.1"}, at); changed { + if _, changed := a.WithNumbered("mesh-sdk", "other", map[string]string{"go": "0.1.15-feat-x-y.1"}, nil, at); changed { t.Error("an ask is marked numbered by a build of another head") } - if _, changed := u.WithNumbered("o", "", map[string]string{"go": "1"}, at); changed { + if _, changed := u.WithNumbered("o", "", map[string]string{"go": "1"}, nil, at); changed { t.Error("an untag is marked numbered") } - n, changed := a.WithNumbered("mesh-sdk", "abc", map[string]string{"go": "0.1.15-feat-x-y.1"}, at.Add(time.Minute)) + n, changed := a.WithNumbered("mesh-sdk", "abc", map[string]string{"go": "0.1.15-feat-x-y.1"}, nil, at.Add(time.Minute)) if !changed || n.State != TagNumbered || n.Versions["mesh-sdk/go"] != "0.1.15-feat-x-y.1" || n.Unanswered() || !strings.Contains(n.Answer, "mesh-sdk/go 0.1.15-feat-x-y.1") { t.Errorf("WithNumbered = %+v, %v", n, changed) } - if _, again := n.WithNumbered("mesh-sdk", "abc", map[string]string{"go": "0.1.15-feat-x-y.1"}, at); again { + if _, again := n.WithNumbered("mesh-sdk", "abc", map[string]string{"go": "0.1.15-feat-x-y.1"}, nil, at); again { t.Error("the same versions mark the ask again") } said := NarrateUnanswered(a, []FloorWait{{Name: "mesh-sdk/go", Silent: []string{"bridge_tags"}}}, at.Add(6*time.Minute)) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 06feccae..3b4a149d 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -100,7 +100,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.packages"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 00b21ca3..113172d8 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -87,7 +87,8 @@ "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules" + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.packages" ], "uses": [ "node-build-go",