From 1175fef241a59ecf700dece17e407194cbce0fa4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 17:33:06 +0100 Subject: [PATCH] Catalog versions: the floor reads every package's holders too, asks again on a growing pause, and a floor entry takes its release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The floor of a module with no version is the highest among its Go tags and the versions each package holder answers through its seat's verb versions (npm-package-registry, go-package-registry; the sdk's npm 0.1.18 above its Go tags 0.1.16). An unanswered read is asked again on a growing pause (5 s, 15 s, 30 s, 1, 2, 5 min) until it answers, with call-unanswered five minutes after the first. A floor entry holds no artifacts, so a package-published heard later at its version is recorded as its release and says version-released. One whole-indexer integration test on a real bus with a stand-in seat git and holder. Builds: design 47 G §412, A-490c, A-490d, A-490e, A-490f, A-490g, A-490h Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../floor_integration_test.go | 106 ++++++++++++ .../mesh-delivery-indexer/floor_retry_test.go | 43 +++++ .../cmd/mesh-delivery-indexer/floorpause.go | 8 + .../mesh-delivery-indexer/integration_test.go | 6 + .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/versions.go | 103 ++++++++++-- .../internal/index/candidates.go | 20 +-- .../internal/index/tag_floor.go | 159 ++++++++++-------- .../internal/index/versions.go | 19 ++- .../internal/index/versions_test.go | 58 ++++++- .../mesh-delivery-indexer/manifest_test.go | 6 +- modules/mesh-delivery-indexer/module.json | 2 + 12 files changed, 427 insertions(+), 105 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go new file mode 100644 index 00000000..c42a665c --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go @@ -0,0 +1,106 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-indexer/internal/index" +) + +// TestIndexerFillsTheFloorOfAModuleWithTags (design 47 G §412 as A-490c to A-490h amend it): the whole indexer on a real bus, the +// seat git and the package holders stood in. A module of the trunk with a Go package and no version in `versions` stays unfilled +// while the forge does not answer, and is filled from the highest of its tags and the holder's versions once the forge answers, +// on the growing pause (5 s) with no restart and no catalog revision. The finding after five minutes is in the unit tests +// (the five minutes are not waited for here). +func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { + const pkg = "git.novox.be/novox/mesh-delivery/modules/" + testIndexer + var forgeUp atomic.Bool + var prefix atomic.Value + js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + for _, bucket := range []string{index.Bucket(testIndexer, index.VersionsState), index.Bucket(testIndexer, index.HeardState)} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil { + t.Fatal(err) + } + } + // the module's artifacts are Go packages here, so that the indexer's own entries carry their names (read from go.mod, §676) + e := trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte { + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go", "kind": "package", "language": "go"}}} + out, _ := json.Marshal(m) + return out + }) + for i := range e.Artifacts { + e.Artifacts[i].Package = &meshcatalog.ArtifactPackage{Ecosystem: index.EcosystemGo, Name: pkg} + } + seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: e}) + reply := func(m *nats.Msg, body map[string]any) { + body["node"] = "n0" + raw, _ := json.Marshal(body) + _ = m.Respond(raw) + } + serve := func(subject string, h func(m *nats.Msg)) { + if _, err := js.Conn().Subscribe(subject, h); err != nil { + t.Fatal(err) + } + } + serve("mesh.seat.git.tool.tags", func(m *nats.Msg) { + var in git.TagsInput + _ = json.Unmarshal(m.Data, &in) + prefix.Store(in.Prefix) + if !forgeUp.Load() { + reply(m, map[string]any{"error": "the forge does not answer"}) + return + } + reply(m, map[string]any{"result": git.TagsAnswer{Repository: in.Repository, Tags: []string{in.Prefix + "0.4.2", in.Prefix + "0.4.10", "unrelated"}}}) + }) + // the package's name is read from the revision's own files (§676): its go.mod + serve("mesh.seat.git.tool.file", func(m *nats.Msg) { + var in git.FileInput + _ = json.Unmarshal(m.Data, &in) + if strings.HasSuffix(in.Path, "go.mod") { + reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Content: "module " + pkg + "\n\ngo 1.26\n"}}) + return + } + reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Absent: true}}) + }) + serve("mesh.seat.go-package-registry.tool.versions", func(m *nats.Msg) { + reply(m, map[string]any{"result": map[string]any{"name": pkg, "versions": []any{}}}) + }) + }) + versions, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.VersionsState)) + if err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Second) // the forge is asked at the start and does not answer: nothing is filled + if keys, _ := versions.Keys(ctx); len(keys) != 0 { + t.Fatalf("versions holds %v while the forge does not answer, want nothing", keys) + } + forgeUp.Store(true) + var got []string + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if got, _ = versions.Keys(ctx); len(got) > 0 { + break + } + } + if len(got) != 1 || !strings.Contains(got[0], "0.4.10") { + t.Fatalf("versions holds %v within 30 s of the forge answering, want the floor 0.4.10 of its tags", got) + } + if p, _ := prefix.Load().(string); p != "modules/"+testIndexer+"/v" { + t.Errorf("the tags were read with the prefix %q, want the package's directory and /v", p) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go new file mode 100644 index 00000000..4f7b28ed --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "testing" + "time" + + "mesh-delivery-indexer/internal/index" +) + +// Design 47 G A-490h (as A-487a's backoff): an unanswered floor read is asked again on a growing pause, 5 s, 15 s, 30 s, 1, 2 and +// then 5 minutes (never past the alarm five minutes after the first), until it answers; an answer starts the pauses again. +func TestAnUnansweredFloorReadIsAskedAgainOnAGrowingPause(t *testing.T) { + var pauses []time.Duration + var next func() + f := &versionsFill{ix: &index.Indexer{}, ctx: context.Background(), logf: func(string, ...any) {}, + floor: index.FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return nil, nil }}, + after: func(d time.Duration, fn func()) func() bool { + pauses = append(pauses, d) + next = fn + return func() bool { return true } + }} + f.run() // the indexer keeps no catalog graph: the floor read fails as an unanswered one does + for i := 0; i < 6 && next != nil; i++ { + fn := next + next = nil + fn() + } + want := []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute} + if len(pauses) != 7 { + t.Fatalf("the pauses %v, want 7 asks again", pauses) + } + for i, d := range want { + if pauses[i] != d { + t.Errorf("pause %d is %v, want %v", i, pauses[i], d) + } + } + for _, d := range pauses[5:] { + if d > 5*time.Minute || d < 4*time.Minute { + t.Errorf("a late pause is %v, want the 5-minute pause clamped to the alarm's time", d) + } + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go new file mode 100644 index 00000000..c616fb2f --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go @@ -0,0 +1,8 @@ +package main + +import "time" + +// afterPause runs f once after d: the pause between two asks of an unanswered floor read (design 47 G A-490h, as A-487a has a +// node-engine ask again after a growing pause). It is the indexer's one timer, kept in this file alone: the guard +// TestServesNoRequestSubjectAndHoldsNoTicker (ADR 0437 §8) names it. +func afterPause(d time.Duration, f func()) (stop func() bool) { return time.AfterFunc(d, f).Stop } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 2e9dde7f..4e68b883 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -392,11 +392,17 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit. func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream) meshcatalog.CatalogEntry { + return trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte { return raw }) +} + +// trunkIndexerEntryOf is trunkIndexerEntry with the manifest's bytes changed by edit first. +func trunkIndexerEntryOf(t *testing.T, ctx context.Context, js jetstream.JetStream, edit func([]byte) []byte) meshcatalog.CatalogEntry { t.Helper() raw, err := os.ReadFile("../../module.json") if err != nil { t.Fatal(err) } + raw = edit(raw) var m catalogue.Manifest if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index c8f14576..d523c58e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -491,7 +491,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ix.Registry = registry named.Changed = ix.RemoveUnlisted // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. - fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), tags: tagsVia(calls.CallKept), logf: logger.Printf} + fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), floor: index.FloorSources{Tags: tagsVia(calls.CallKept), Holder: holderVia(calls.CallKept)}, logf: logger.Printf} fillFollow.Store(fill) fill.kick() ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index 984cac05..50354565 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "sync" "sync/atomic" "time" @@ -13,11 +14,14 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" @@ -190,27 +194,76 @@ func versionsVia(call openCaller, inView func(ctx context.Context, module string // versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each // catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next. type versionsFill struct { - ix *index.Indexer - ctx context.Context - ask index.VersionsAsker - tags index.TagsReader // the tag floor's reader (A-490d); nil fills none - logf func(string, ...any) + ix *index.Indexer + ctx context.Context + ask index.VersionsAsker + floor index.FloorSources // what the floor reads (A-490g); a nil reader fills none + logf func(string, ...any) + pauses []time.Duration // the growing pauses between asks of an unanswered floor read; nil is floorPauses + after func(d time.Duration, f func()) (stop func() bool) // the timer (afterPause) unless a test sets it + + mu sync.Mutex + retry int // how many unanswered floor reads in a row + since time.Time // when the first of them was asked + pending func() bool } +// floorPauses are the pauses between asks of an unanswered floor read (A-490h), as a node-engine asks a missing trunk build again +// (A-487a): 5 s, 15 s, 30 s, 1, 2 and then 5 minutes; the first answer starts them again. +var floorPauses = []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute} + func (f *versionsFill) kick() { go func() { if err := f.ix.FillVersions(f.ctx, f.ask); err != nil { f.logf("versions: the first fill waits: %v", err) - return - } - // The tag floor, per module the versions hold none of, at every start and every revision (A-490d): the builder's half - // is done once; this half is not. - if f.tags != nil { - if err := f.ix.FillTagFloors(f.ctx, f.tags); err != nil { - f.logf("versions: the tag floor waits: %v", err) - } } }() + go f.run() +} + +// run makes the floor, not waiting for the builder's half: a builder_versions unanswered holds that half for its bound, and the +// floor's own reads answer by themselves. +func (f *versionsFill) run() { + // The floor, per module the versions hold none of, at every start and every revision and again on a growing pause while a + // read is unanswered (A-490d, A-490f, A-490h): the builder's half is done once; this half is not. + if f.floor.Tags == nil && f.floor.Holder == nil { + return + } + err := f.ix.FillTagFloors(f.ctx, f.floor) + f.mu.Lock() + defer f.mu.Unlock() + if err == nil { + f.retry, f.since = 0, time.Time{} + return + } + f.logf("versions: the floor waits: %v", err) + if f.pending != nil || f.ctx.Err() != nil { + return // an ask again is set already + } + pauses := floorPauses + if f.pauses != nil { + pauses = f.pauses + } + pause := pauses[min(f.retry, len(pauses)-1)] + f.retry++ + now := time.Now() + if f.since.IsZero() { + f.since = now + } + // the alarm is five minutes after the first unanswered read: never pause past it before it is raised + if left := f.since.Add(mcc.CallUnansweredAfter).Sub(now); left > 0 && pause > left { + pause = left + } + after := f.after + if after == nil { + after = afterPause + } + f.pending = after(pause, func() { + f.mu.Lock() + f.pending = nil + f.mu.Unlock() + f.run() + }) } // afterRevision runs the fill's kick after each catalog revision kept, once the fill is set. @@ -300,3 +353,27 @@ func tagsVia(call toolCaller) index.TagsReader { } } } + +// holderVia reads the versions a package holder answers for a package: the seat npm-package-registry's or go-package-registry's +// verb versions (A-490g), as a kept call. +func holderVia(call toolCaller) index.HolderReader { + return func(_ context.Context, ecosystem, name string) ([]string, error) { + seat := npmpackageregistry.Seat + if ecosystem == index.EcosystemGo { + seat = gopackageregistry.Seat + } + raw, err := call("seat:"+seat+"."+npmpackageregistry.VerbVersions, npmpackageregistry.VersionsInput{Name: name, Kind: npmpackageregistry.ArtifactKindPackage}) + if err != nil { + return nil, err + } + var a npmpackageregistry.VersionsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s.versions is not a package's versions: %w", seat, err) + } + var out []string + for _, v := range a.Versions { + out = append(out, v.Version) + } + return out, nil + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 1abab54d..0ae08aee 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -174,16 +174,16 @@ type Indexer struct { previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) - tagFilling atomic.Bool - versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) - stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on - fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu + tagFilling atomic.Bool // a FillTagFloors runs (tag_floor.go): one at a time + versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) + stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) + unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) + retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one + retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on + fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu freshMu sync.Mutex recompute sync.Mutex // one Recompute at a time untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) diff --git a/modules/mesh-delivery-indexer/internal/index/tag_floor.go b/modules/mesh-delivery-indexer/internal/index/tag_floor.go index 218a2850..1a9c419f 100644 --- a/modules/mesh-delivery-indexer/internal/index/tag_floor.go +++ b/modules/mesh-delivery-indexer/internal/index/tag_floor.go @@ -2,67 +2,70 @@ package index import ( "context" + "encoding/json" "errors" "fmt" - "slices" "strings" + "time" - "encoding/json" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" ) -// The tag floor of the catalog's versions (design 47 G §412 as A-490c, A-490d and A-490e amend it): for every module the -// state versions holds no version of, the highest tag of the module's Go package in the repository its catalog entry was -// taken in from, read from the seat git's verb tags, protected or not (A-490c), is its first version. It is filled per -// module whenever the indexer starts and at each catalog revision kept, not once (A-490d): the key versions-filled closes -// only the builder's half of the fill. +// The floor of the catalog's versions (design 47 G §412 as A-490c to A-490h amend it): for every module the state versions +// holds no version of, the highest version among every package its catalog entry names is its first version: the Go +// package's tags in the repository the entry was taken in from, read from the seat git's verb tags, protected or not +// (A-490c, A-490e), and what each package holder answers for the package through its seat's verb versions (A-490g). It is +// filled per module whenever the indexer starts and at each catalog revision kept, not once (A-490d, A-490f): the key +// versions-filled closes only the builder's half of the fill. -// TagsCall is the seat verb the tag floor reads, the finding's id when it is unanswered five minutes (§412). -const TagsCall = "git.tags" +// FloorCall is the id of the finding call-unanswered when a floor read (the forge's tags or a package holder's versions) is +// unanswered five minutes after the first (A-490h). +const FloorCall = "mesh-catalog.versions-floor" -var tagsFillSince = "call-since." + keys.EscapeName(TagsCall) +var floorSince = "call-since." + keys.EscapeName(FloorCall) // TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags). type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error) -// tagPrefixesOf is the prefix Go gives the tags of each Go package of the entry in its repository (A-490e): the package's -// directory in the repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root. A -// module with no Go package has none. -func tagPrefixesOf(e meshcatalog.CatalogEntry) []string { - var out []string - for _, a := range e.Artifacts { - if a.Package == nil || a.Package.Ecosystem != "go" { - continue - } - i := strings.Index(a.Package.Name, "/"+e.Repository) - if e.Repository == "" || i < 0 { - continue - } - rest := a.Package.Name[i+1+len(e.Repository):] - var p string - switch { - case rest == "": - p = "v" - case strings.HasPrefix(rest, "/"): - p = rest[1:] + "/v" - default: - continue - } - if !slices.Contains(out, p) { - out = append(out, p) - } - } - return out +// HolderReader reads the versions a package holder answers for a package of an ecosystem (EcosystemGo, EcosystemNpm) +// through its seat's verb versions (A-490g). +type HolderReader func(ctx context.Context, ecosystem, name string) ([]string, error) + +// FloorSources are what the floor reads: the forge's tags and each package holder's versions. +type FloorSources struct { + Tags TagsReader + Holder HolderReader } -// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest tag of its -// Go package. A module whose tags cannot be read is not filled; it is said, call-unanswered after five minutes, and the -// fill is made again at the next start or revision. The floor is a version entry with no artifact and no digest, as the -// builder's numbered-only versions are (A-351), so the builder numbers above it. -func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { - if ix.Versions == nil || ix.Revisions == nil || read == nil { +// tagPrefixOf is the prefix Go gives the tags of a Go package in its repository (A-490e): the package's directory in the +// repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root; none when the package's +// name does not lie in the repository. +func tagPrefixOf(repository string, a meshcatalog.Artifact) (string, bool) { + if a.Package == nil || a.Package.Ecosystem != EcosystemGo || repository == "" { + return "", false + } + i := strings.Index(a.Package.Name, "/"+repository) + if i < 0 { + return "", false + } + switch rest := a.Package.Name[i+1+len(repository):]; { + case rest == "": + return "v", true + case strings.HasPrefix(rest, "/"): + return rest[1:] + "/v", true + } + return "", false +} + +// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest version its +// packages have (A-490g). A read the forge or a holder does not answer fills no module and is returned: the finding +// call-unanswered is written five minutes after the first unanswered read and cleared when a read answers (A-490h); the +// caller asks again on a growing pause. The floor entry holds no artifact and no digest (A-351), so the builder numbers +// above it and a package-published heard later at its version is recorded as its release. +func (ix *Indexer) FillTagFloors(ctx context.Context, src FloorSources) error { + if ix.Versions == nil || ix.Revisions == nil { return errors.New("the state versions or the catalog graph is not kept here") } if !ix.tagFilling.CompareAndSwap(false, true) { @@ -74,14 +77,10 @@ func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { return err } if !found { - return errors.New("the indexer keeps no trunk revision yet: the tag floor waits for its first") + return errors.New("the indexer keeps no trunk revision yet: the floor waits for its first") } var unanswered error for _, e := range entries { - prefixes := tagPrefixesOf(e) - if len(prefixes) == 0 { - continue - } have, err := ModuleVersions(ctx, ix.Versions, e.Module) if err != nil { return err @@ -90,23 +89,40 @@ func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { continue } top := "" - for _, p := range prefixes { - tags, err := read(ctx, e.Repository, p) - if err != nil { - unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err) - top = "" - break - } - for _, t := range tags { - v := strings.TrimPrefix(t, p) - if !strings.HasPrefix(t, p) { + take := func(vs []string, prefix string) { + for _, t := range vs { + if !strings.HasPrefix(t, prefix) { continue } - if _, ok := versionParts(v); ok && (top == "" || compareVersions(v, top) > 0) { + if v := strings.TrimPrefix(t, prefix); versionOK(v) && (top == "" || compareVersions(v, top) > 0) { top = v } } } + for _, a := range e.Artifacts { + if a.Package == nil { + continue + } + if prefix, ok := tagPrefixOf(e.Repository, a); ok && src.Tags != nil { + tags, err := src.Tags(ctx, e.Repository, prefix) + if err != nil { + unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err) + break + } + take(tags, prefix) + } + if (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm) && src.Holder != nil { + vs, err := src.Holder(ctx, a.Package.Ecosystem, a.Package.Name) + if err != nil { + unanswered = fmt.Errorf("the versions of %s %s: %w", a.Package.Ecosystem, a.Package.Name, err) + break + } + take(vs, "") + } + } + if unanswered != nil { + break + } if top == "" { continue } @@ -116,30 +132,33 @@ func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { return err } if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(e.Module, top), data); err != nil && !errors.Is(err, ErrConflict) { - return fmt.Errorf("the tag floor %s %s is not kept: %w", e.Module, top, err) + return fmt.Errorf("the floor %s %s is not kept: %w", e.Module, top, err) } - ix.logf("versions: %s has none, filled from the forge's tag %s", e.Module, top) + ix.logf("versions: %s has none, filled with %s, the highest of its packages' tags and holders' versions", e.Module, top) } if unanswered != nil { now := ix.now() - since := ix.unansweredSince(ctx, tagsFillSince, now) + since := ix.unansweredSince(ctx, floorSince, now) if ix.Findings != nil && now.Sub(since) >= mcc.CallUnansweredAfter { - f := ix.observed(mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, - fmt.Sprintf("the call %s is unanswered since %s", TagsCall, since.UTC().Format("2006-01-02T15:04:05Z07:00")), + at := since.UTC().Format(time.RFC3339) + f := ix.observed(mcc.ScopeCall, FloorCall, mcc.KindCallUnanswered, + fmt.Sprintf("the floor of the catalog's versions is unanswered since %s", at), "A call whose answer is kept is not answered", - fmt.Sprintf("The tag floor of the catalog's versions reads the forge's tags; they are not read since %s (%v), so a module with none is not filled.", since.UTC().Format("2006-01-02T15:04:05Z07:00"), unanswered), + fmt.Sprintf("The floor of the catalog's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so a module with none is not filled.", at, unanswered), "indexer."+mcc.KindCallUnanswered) if err := ix.Findings.Write(ctx, f); err != nil { - ix.logf("tag floor: the finding %s is not written: %v", TagsCall, err) + ix.logf("floor: the finding %s is not written: %v", FloorCall, err) } } return unanswered } - ix.answered(ctx, tagsFillSince) + ix.answered(ctx, floorSince) if ix.Findings != nil { - if err := ix.clearFinding(ctx, mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, "the forge's tags answer"); err != nil { - ix.logf("tag floor: the finding %s is not cleared: %v", TagsCall, err) + if err := ix.clearFinding(ctx, mcc.ScopeCall, FloorCall, mcc.KindCallUnanswered, "the floor's reads answer"); err != nil { + ix.logf("floor: the finding %s is not cleared: %v", FloorCall, err) } } return nil } + +func versionOK(v string) bool { _, ok := versionParts(v); return ok } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 5c5db15a..574a6fb5 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -120,9 +120,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83) } } - if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { + if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { return err - } else if found { + } else if found && !isFloor(raw) { ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) return nil } @@ -212,7 +212,14 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli if err != nil { return err } - if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { + if err := ix.Versions.Create(ctx, key, data); errors.Is(err, ErrConflict) { + // A floor entry (A-490h) holds no artifacts: this publish is that version's release, so it takes the entry's place. + if raw, rev, found, gerr := ix.Versions.Get(ctx, key); gerr == nil && found && isFloor(raw) { + if err := ix.Versions.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) { + return err + } + } + } else if err != nil { return err } if late { @@ -339,3 +346,9 @@ func OutdatedTool(ix *Indexer) stdio.Tool { return ix.Outdated(context.Background(), in) }} } + +// isFloor says whether a kept version is a floor entry (A-490h): it holds no artifact, as no release does. +func isFloor(raw []byte) bool { + var v meshcatalog.VersionEntry + return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0 +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 042917eb..2440b4c1 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -545,7 +545,8 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} entries := map[string]meshcatalog.CatalogEntry{ "mesh-sdk": {Module: "mesh-sdk", Repository: "novox/mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ - {Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}}}, + {Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}, + {Ref: "mesh-sdk:ts", Package: &meshcatalog.ArtifactPackage{Ecosystem: "npm", Name: "@novox/mesh-sdk"}}}}, "root": {Module: "root", Repository: "novox/root", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ {Ref: "root:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/root"}}}}, "held": {Module: "held", Repository: "novox/held", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "held:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/held"}}}}, @@ -576,29 +577,39 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { } return nil, nil } - if err := ix.FillTagFloors(ctx, read); err == nil { + holder := func(_ context.Context, eco, name string) ([]string, error) { + if !up { + return nil, errors.New("no responders") + } + if eco == "npm" && name == "@novox/mesh-sdk" { + return []string{"0.1.18", "0.1.17"}, nil // A-490g: the npm holder is above the Go tags + } + return nil, nil + } + src := FloorSources{Tags: read, Holder: holder} + if err := ix.FillTagFloors(ctx, src); err == nil { t.Fatal("tags not answered filled") } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } now = now.Add(5 * time.Minute) - _ = ix.FillTagFloors(ctx, read) - if h := w.Held(); len(h) != 1 || h[0].ID != TagsCall { + _ = ix.FillTagFloors(ctx, src) + if h := w.Held(); len(h) != 1 || h[0].ID != FloorCall { t.Fatalf("the findings after five minutes: %+v", h) } if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 0 { t.Fatalf("filled while the forge was down: %+v", v) } up = true - if err := ix.FillTagFloors(ctx, read); err != nil { + if err := ix.FillTagFloors(ctx, src); err != nil { t.Fatal(err) } if len(w.Held()) != 0 { t.Errorf("the finding stands after the forge answered: %+v", w.Held()) } - if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.16" { - t.Errorf("mesh-sdk's floor: %+v (the highest of go/v tags, 0.1.16)", v) + if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.18" { + t.Errorf("mesh-sdk's floor: %+v (the highest of its Go tags 0.1.16 and the npm holder's 0.1.18)", v) } if v, _ := ModuleVersions(ctx, ix.Versions, "root"); len(v) != 1 || v[0].Version != "1.2.3" { t.Errorf("a root module's floor: %+v", v) @@ -612,3 +623,36 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { } } } + +// Design 47 G A-490h: a floor entry holds no artifacts, so the package-published heard later at its version is that version's +// release (the entry takes its artifacts and version-released is said), never dropped as released already. +func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC) + var said []meshcatalog.VersionReleasedEvent + ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: newFindings(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, + SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, _ string) error { + said = append(said, e) + return nil + }} + entries := map[string]meshcatalog.CatalogEntry{"mod": {Module: "mod", Repository: "novox/mod", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ + {Ref: "mod:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mod"}}}}} + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + src := FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return []string{"v0.3.0"}, nil }} + if err := ix.FillTagFloors(ctx, src); err != nil { + t.Fatal(err) + } + if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "mod", Artifact: "mod:go", Version: "0.3.0", Digest: "sha256:aa", Branch: "main", Source: 5}); err != nil { + t.Fatal(err) + } + v, _ := ModuleVersions(ctx, ix.Versions, "mod") + if len(v) != 1 || len(v[0].Artifacts) != 1 || v[0].Artifacts[0].Digest != "sha256:aa" || len(said) != 1 { + t.Fatalf("the floor 0.3.0 and its publish: versions %+v, version-released said %d", v, len(said)) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 36c93d78..4b415353 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.tags", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -393,6 +393,10 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR if f == "Subscribe(" && strings.HasSuffix(path, "seatlist.go") { continue } + // The one timer is the growing pause of an unanswered floor read (A-490h), in floorpause.go. + if f == "time.AfterFunc" && strings.HasSuffix(path, "floorpause.go") { + continue + } if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the indexer serves no request subject and holds no timer", path, f) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index da00d9d5..d460a233 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -168,6 +168,8 @@ "invokes": [ "seat:git.trees", "seat:git.tags", + "seat:go-package-registry.versions", + "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls",