From ea4fe471e2d3ad8a9eab95ec2f008f07fee0349e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 17:11:15 +0100 Subject: [PATCH 1/4] Catalog versions: fill the tag floor per module with no version, from the forge's Go tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh-catalog.versions answered [] for mesh-sdk although the forge holds go/v0.1.12..v0.1.16: the first fill read only builder_versions, so a module the builder held nothing of was filled with nothing, and the once-only key closed it. The indexer now reads, for every module of the newest trunk revision the versions hold none of, every tag of its Go package from the seat git's verb tags (repository of its catalog entry, prefix the package's directory + "/v", "v" at the root), fills the highest as its version, at each start and each catalog revision kept, and raises call-unanswered after five minutes when the forge does not answer. Builds: design 47 G §412, A-490b, A-490c, A-490d, A-490e Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/versions.go | 34 ++++ .../internal/index/candidates.go | 19 +-- .../internal/index/tag_floor.go | 145 ++++++++++++++++++ .../internal/index/versions_test.go | 78 ++++++++++ .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 1 + 7 files changed, 271 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/tag_floor.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 9bc84e90..c8f14576 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -491,7 +491,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ix.Registry = registry named.Changed = ix.RemoveUnlisted // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. - fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), logf: logger.Printf} + fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), tags: tagsVia(calls.CallKept), logf: logger.Printf} fillFollow.Store(fill) fill.kick() ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index cb8076b2..984cac05 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "sync/atomic" "time" @@ -12,10 +13,12 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" @@ -190,6 +193,7 @@ type versionsFill struct { ix *index.Indexer ctx context.Context ask index.VersionsAsker + tags index.TagsReader // the tag floor's reader (A-490d); nil fills none logf func(string, ...any) } @@ -197,6 +201,14 @@ func (f *versionsFill) kick() { go func() { if err := f.ix.FillVersions(f.ctx, f.ask); err != nil { f.logf("versions: the first fill waits: %v", err) + return + } + // The tag floor, per module the versions hold none of, at every start and every revision (A-490d): the builder's half + // is done once; this half is not. + if f.tags != nil { + if err := f.ix.FillTagFloors(f.ctx, f.tags); err != nil { + f.logf("versions: the tag floor waits: %v", err) + } } }() } @@ -266,3 +278,25 @@ func runningWholeVia(js jetstream.JetStream) index.RunningReader { } } } + +// tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by +// page until none is more (design 47 G §412, A-490c: protected or not). +func tagsVia(call toolCaller) index.TagsReader { + return func(_ context.Context, repository, prefix string) ([]string, error) { + var out []string + for page := int64(1); ; page++ { + var a git.TagsAnswer + in := git.TagsInput{Repository: trunk.RepositoryName(repository), Prefix: prefix, Page: page} + if !strings.Contains(in.Repository, "/") { + return nil, fmt.Errorf("%q is not /", repository) + } + if err := askGit(call, git.VerbTags, in, &a); err != nil { + return nil, err + } + out = append(out, a.Tags...) + if !a.More { + return out, nil + } + } + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 1390342c..1abab54d 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -174,15 +174,16 @@ type Indexer struct { previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) - versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) - stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on - fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu + tagFilling atomic.Bool + versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) + stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) + unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) + retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one + retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on + fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu freshMu sync.Mutex recompute sync.Mutex // one Recompute at a time untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) diff --git a/modules/mesh-delivery-indexer/internal/index/tag_floor.go b/modules/mesh-delivery-indexer/internal/index/tag_floor.go new file mode 100644 index 00000000..218a2850 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/tag_floor.go @@ -0,0 +1,145 @@ +package index + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + "encoding/json" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The tag floor of the catalog's versions (design 47 G §412 as A-490c, A-490d and A-490e amend it): for every module the +// state versions holds no version of, the highest tag of the module's Go package in the repository its catalog entry was +// taken in from, read from the seat git's verb tags, protected or not (A-490c), is its first version. It is filled per +// module whenever the indexer starts and at each catalog revision kept, not once (A-490d): the key versions-filled closes +// only the builder's half of the fill. + +// TagsCall is the seat verb the tag floor reads, the finding's id when it is unanswered five minutes (§412). +const TagsCall = "git.tags" + +var tagsFillSince = "call-since." + keys.EscapeName(TagsCall) + +// TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags). +type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error) + +// tagPrefixesOf is the prefix Go gives the tags of each Go package of the entry in its repository (A-490e): the package's +// directory in the repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root. A +// module with no Go package has none. +func tagPrefixesOf(e meshcatalog.CatalogEntry) []string { + var out []string + for _, a := range e.Artifacts { + if a.Package == nil || a.Package.Ecosystem != "go" { + continue + } + i := strings.Index(a.Package.Name, "/"+e.Repository) + if e.Repository == "" || i < 0 { + continue + } + rest := a.Package.Name[i+1+len(e.Repository):] + var p string + switch { + case rest == "": + p = "v" + case strings.HasPrefix(rest, "/"): + p = rest[1:] + "/v" + default: + continue + } + if !slices.Contains(out, p) { + out = append(out, p) + } + } + return out +} + +// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest tag of its +// Go package. A module whose tags cannot be read is not filled; it is said, call-unanswered after five minutes, and the +// fill is made again at the next start or revision. The floor is a version entry with no artifact and no digest, as the +// builder's numbered-only versions are (A-351), so the builder numbers above it. +func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { + if ix.Versions == nil || ix.Revisions == nil || read == nil { + return errors.New("the state versions or the catalog graph is not kept here") + } + if !ix.tagFilling.CompareAndSwap(false, true) { + return nil + } + defer ix.tagFilling.Store(false) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil { + return err + } + if !found { + return errors.New("the indexer keeps no trunk revision yet: the tag floor waits for its first") + } + var unanswered error + for _, e := range entries { + prefixes := tagPrefixesOf(e) + if len(prefixes) == 0 { + continue + } + have, err := ModuleVersions(ctx, ix.Versions, e.Module) + if err != nil { + return err + } + if len(have) > 0 { + continue + } + top := "" + for _, p := range prefixes { + tags, err := read(ctx, e.Repository, p) + if err != nil { + unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err) + top = "" + break + } + for _, t := range tags { + v := strings.TrimPrefix(t, p) + if !strings.HasPrefix(t, p) { + continue + } + if _, ok := versionParts(v); ok && (top == "" || compareVersions(v, top) > 0) { + top = v + } + } + } + if top == "" { + continue + } + now := ix.now().UTC() + data, err := json.Marshal(meshcatalog.VersionEntry{V: int64(keys.Version), Module: e.Module, Version: top, At: &now}) + if err != nil { + return err + } + if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(e.Module, top), data); err != nil && !errors.Is(err, ErrConflict) { + return fmt.Errorf("the tag floor %s %s is not kept: %w", e.Module, top, err) + } + ix.logf("versions: %s has none, filled from the forge's tag %s", e.Module, top) + } + if unanswered != nil { + now := ix.now() + since := ix.unansweredSince(ctx, tagsFillSince, now) + if ix.Findings != nil && now.Sub(since) >= mcc.CallUnansweredAfter { + f := ix.observed(mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, + fmt.Sprintf("the call %s is unanswered since %s", TagsCall, since.UTC().Format("2006-01-02T15:04:05Z07:00")), + "A call whose answer is kept is not answered", + fmt.Sprintf("The tag floor of the catalog's versions reads the forge's tags; they are not read since %s (%v), so a module with none is not filled.", since.UTC().Format("2006-01-02T15:04:05Z07:00"), unanswered), + "indexer."+mcc.KindCallUnanswered) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("tag floor: the finding %s is not written: %v", TagsCall, err) + } + } + return unanswered + } + ix.answered(ctx, tagsFillSince) + if ix.Findings != nil { + if err := ix.clearFinding(ctx, mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, "the forge's tags answer"); err != nil { + ix.logf("tag floor: the finding %s is not cleared: %v", TagsCall, err) + } + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 74115e91..042917eb 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -534,3 +534,81 @@ func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) { t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err) } } + +// Design 47 G §412 as A-490c, A-490d and A-490e amend it: a module the versions hold none of is filled from the highest tag of its +// Go package in the repository it was taken in from, per module and not once; a forge that does not answer fills nothing and +// raises call-unanswered after five minutes. +func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC) + w := newFindings() + ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} + entries := map[string]meshcatalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk", Repository: "novox/mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ + {Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}}}, + "root": {Module: "root", Repository: "novox/root", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ + {Ref: "root:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/root"}}}}, + "held": {Module: "held", Repository: "novox/held", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "held:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/held"}}}}, + "image": {Module: "image", Repository: "novox/image", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "image:image", Kind: "image"}}}, + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("held", "0.2.0"), []byte(`{"v":1,"module":"held","version":"0.2.0"}`)); err != nil { + t.Fatal(err) + } + up := false + var asked []string + read := func(_ context.Context, repo, prefix string) ([]string, error) { + asked = append(asked, repo+" "+prefix) + if !up { + return nil, errors.New("no responders") + } + switch repo + " " + prefix { + case "novox/mesh-sdk go/v": + return []string{"go/v0.1.9", "go/v0.1.16", "go/v0.1.12", "go/vnext"}, nil + case "novox/root v": + return []string{"v1.2.3", "vendor-x"}, nil + } + return nil, nil + } + if err := ix.FillTagFloors(ctx, read); err == nil { + t.Fatal("tags not answered filled") + } + if len(w.Held()) != 0 { + t.Fatal("a call unanswered for no time wrote its finding") + } + now = now.Add(5 * time.Minute) + _ = ix.FillTagFloors(ctx, read) + if h := w.Held(); len(h) != 1 || h[0].ID != TagsCall { + t.Fatalf("the findings after five minutes: %+v", h) + } + if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 0 { + t.Fatalf("filled while the forge was down: %+v", v) + } + up = true + if err := ix.FillTagFloors(ctx, read); err != nil { + t.Fatal(err) + } + if len(w.Held()) != 0 { + t.Errorf("the finding stands after the forge answered: %+v", w.Held()) + } + if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.16" { + t.Errorf("mesh-sdk's floor: %+v (the highest of go/v tags, 0.1.16)", v) + } + if v, _ := ModuleVersions(ctx, ix.Versions, "root"); len(v) != 1 || v[0].Version != "1.2.3" { + t.Errorf("a root module's floor: %+v", v) + } + if v, _ := ModuleVersions(ctx, ix.Versions, "held"); len(v) != 1 || v[0].Version != "0.2.0" { + t.Errorf("a module with a version is filled: %+v", v) + } + for _, a := range asked { + if strings.HasPrefix(a, "novox/held") || strings.HasPrefix(a, "novox/image") { + t.Errorf("tags read for a module with a version or no Go package: %s", a) + } + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 439a6b0b..36c93d78 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.tags", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -416,7 +416,7 @@ func TestEveryGiteaToolTheCodeCallsIsInInvokes(t *testing.T) { if err != nil || len(files) == 0 { t.Fatalf("no program files: %v", err) } - verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch} + verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch, "VerbTags": git.VerbTags} forge, asked := regexp.MustCompile(`gitea\.gitea_[a-z_]+`), regexp.MustCompile(`git\.Verb[A-Za-z]+`) seen := 0 for _, f := range files { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 49f95a7d..da00d9d5 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -167,6 +167,7 @@ ], "invokes": [ "seat:git.trees", + "seat:git.tags", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", From 1175fef241a59ecf700dece17e407194cbce0fa4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 17:33:06 +0100 Subject: [PATCH 2/4] Catalog versions: the floor reads every package's holders too, asks again on a growing pause, and a floor entry takes its release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The floor of a module with no version is the highest among its Go tags and the versions each package holder answers through its seat's verb versions (npm-package-registry, go-package-registry; the sdk's npm 0.1.18 above its Go tags 0.1.16). An unanswered read is asked again on a growing pause (5 s, 15 s, 30 s, 1, 2, 5 min) until it answers, with call-unanswered five minutes after the first. A floor entry holds no artifacts, so a package-published heard later at its version is recorded as its release and says version-released. One whole-indexer integration test on a real bus with a stand-in seat git and holder. Builds: design 47 G §412, A-490c, A-490d, A-490e, A-490f, A-490g, A-490h Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../floor_integration_test.go | 106 ++++++++++++ .../mesh-delivery-indexer/floor_retry_test.go | 43 +++++ .../cmd/mesh-delivery-indexer/floorpause.go | 8 + .../mesh-delivery-indexer/integration_test.go | 6 + .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/versions.go | 103 ++++++++++-- .../internal/index/candidates.go | 20 +-- .../internal/index/tag_floor.go | 159 ++++++++++-------- .../internal/index/versions.go | 19 ++- .../internal/index/versions_test.go | 58 ++++++- .../mesh-delivery-indexer/manifest_test.go | 6 +- modules/mesh-delivery-indexer/module.json | 2 + 12 files changed, 427 insertions(+), 105 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go new file mode 100644 index 00000000..c42a665c --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go @@ -0,0 +1,106 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-indexer/internal/index" +) + +// TestIndexerFillsTheFloorOfAModuleWithTags (design 47 G §412 as A-490c to A-490h amend it): the whole indexer on a real bus, the +// seat git and the package holders stood in. A module of the trunk with a Go package and no version in `versions` stays unfilled +// while the forge does not answer, and is filled from the highest of its tags and the holder's versions once the forge answers, +// on the growing pause (5 s) with no restart and no catalog revision. The finding after five minutes is in the unit tests +// (the five minutes are not waited for here). +func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { + const pkg = "git.novox.be/novox/mesh-delivery/modules/" + testIndexer + var forgeUp atomic.Bool + var prefix atomic.Value + js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + for _, bucket := range []string{index.Bucket(testIndexer, index.VersionsState), index.Bucket(testIndexer, index.HeardState)} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil { + t.Fatal(err) + } + } + // the module's artifacts are Go packages here, so that the indexer's own entries carry their names (read from go.mod, §676) + e := trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte { + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go", "kind": "package", "language": "go"}}} + out, _ := json.Marshal(m) + return out + }) + for i := range e.Artifacts { + e.Artifacts[i].Package = &meshcatalog.ArtifactPackage{Ecosystem: index.EcosystemGo, Name: pkg} + } + seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: e}) + reply := func(m *nats.Msg, body map[string]any) { + body["node"] = "n0" + raw, _ := json.Marshal(body) + _ = m.Respond(raw) + } + serve := func(subject string, h func(m *nats.Msg)) { + if _, err := js.Conn().Subscribe(subject, h); err != nil { + t.Fatal(err) + } + } + serve("mesh.seat.git.tool.tags", func(m *nats.Msg) { + var in git.TagsInput + _ = json.Unmarshal(m.Data, &in) + prefix.Store(in.Prefix) + if !forgeUp.Load() { + reply(m, map[string]any{"error": "the forge does not answer"}) + return + } + reply(m, map[string]any{"result": git.TagsAnswer{Repository: in.Repository, Tags: []string{in.Prefix + "0.4.2", in.Prefix + "0.4.10", "unrelated"}}}) + }) + // the package's name is read from the revision's own files (§676): its go.mod + serve("mesh.seat.git.tool.file", func(m *nats.Msg) { + var in git.FileInput + _ = json.Unmarshal(m.Data, &in) + if strings.HasSuffix(in.Path, "go.mod") { + reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Content: "module " + pkg + "\n\ngo 1.26\n"}}) + return + } + reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Absent: true}}) + }) + serve("mesh.seat.go-package-registry.tool.versions", func(m *nats.Msg) { + reply(m, map[string]any{"result": map[string]any{"name": pkg, "versions": []any{}}}) + }) + }) + versions, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.VersionsState)) + if err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Second) // the forge is asked at the start and does not answer: nothing is filled + if keys, _ := versions.Keys(ctx); len(keys) != 0 { + t.Fatalf("versions holds %v while the forge does not answer, want nothing", keys) + } + forgeUp.Store(true) + var got []string + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if got, _ = versions.Keys(ctx); len(got) > 0 { + break + } + } + if len(got) != 1 || !strings.Contains(got[0], "0.4.10") { + t.Fatalf("versions holds %v within 30 s of the forge answering, want the floor 0.4.10 of its tags", got) + } + if p, _ := prefix.Load().(string); p != "modules/"+testIndexer+"/v" { + t.Errorf("the tags were read with the prefix %q, want the package's directory and /v", p) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go new file mode 100644 index 00000000..4f7b28ed --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "testing" + "time" + + "mesh-delivery-indexer/internal/index" +) + +// Design 47 G A-490h (as A-487a's backoff): an unanswered floor read is asked again on a growing pause, 5 s, 15 s, 30 s, 1, 2 and +// then 5 minutes (never past the alarm five minutes after the first), until it answers; an answer starts the pauses again. +func TestAnUnansweredFloorReadIsAskedAgainOnAGrowingPause(t *testing.T) { + var pauses []time.Duration + var next func() + f := &versionsFill{ix: &index.Indexer{}, ctx: context.Background(), logf: func(string, ...any) {}, + floor: index.FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return nil, nil }}, + after: func(d time.Duration, fn func()) func() bool { + pauses = append(pauses, d) + next = fn + return func() bool { return true } + }} + f.run() // the indexer keeps no catalog graph: the floor read fails as an unanswered one does + for i := 0; i < 6 && next != nil; i++ { + fn := next + next = nil + fn() + } + want := []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute} + if len(pauses) != 7 { + t.Fatalf("the pauses %v, want 7 asks again", pauses) + } + for i, d := range want { + if pauses[i] != d { + t.Errorf("pause %d is %v, want %v", i, pauses[i], d) + } + } + for _, d := range pauses[5:] { + if d > 5*time.Minute || d < 4*time.Minute { + t.Errorf("a late pause is %v, want the 5-minute pause clamped to the alarm's time", d) + } + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go new file mode 100644 index 00000000..c616fb2f --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go @@ -0,0 +1,8 @@ +package main + +import "time" + +// afterPause runs f once after d: the pause between two asks of an unanswered floor read (design 47 G A-490h, as A-487a has a +// node-engine ask again after a growing pause). It is the indexer's one timer, kept in this file alone: the guard +// TestServesNoRequestSubjectAndHoldsNoTicker (ADR 0437 §8) names it. +func afterPause(d time.Duration, f func()) (stop func() bool) { return time.AfterFunc(d, f).Stop } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 2e9dde7f..4e68b883 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -392,11 +392,17 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 // trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit. func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream) meshcatalog.CatalogEntry { + return trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte { return raw }) +} + +// trunkIndexerEntryOf is trunkIndexerEntry with the manifest's bytes changed by edit first. +func trunkIndexerEntryOf(t *testing.T, ctx context.Context, js jetstream.JetStream, edit func([]byte) []byte) meshcatalog.CatalogEntry { t.Helper() raw, err := os.ReadFile("../../module.json") if err != nil { t.Fatal(err) } + raw = edit(raw) var m catalogue.Manifest if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index c8f14576..d523c58e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -491,7 +491,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ix.Registry = registry named.Changed = ix.RemoveUnlisted // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. - fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), tags: tagsVia(calls.CallKept), logf: logger.Printf} + fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), floor: index.FloorSources{Tags: tagsVia(calls.CallKept), Holder: holderVia(calls.CallKept)}, logf: logger.Printf} fillFollow.Store(fill) fill.kick() ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index 984cac05..50354565 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "sync" "sync/atomic" "time" @@ -13,11 +14,14 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" @@ -190,27 +194,76 @@ func versionsVia(call openCaller, inView func(ctx context.Context, module string // versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each // catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next. type versionsFill struct { - ix *index.Indexer - ctx context.Context - ask index.VersionsAsker - tags index.TagsReader // the tag floor's reader (A-490d); nil fills none - logf func(string, ...any) + ix *index.Indexer + ctx context.Context + ask index.VersionsAsker + floor index.FloorSources // what the floor reads (A-490g); a nil reader fills none + logf func(string, ...any) + pauses []time.Duration // the growing pauses between asks of an unanswered floor read; nil is floorPauses + after func(d time.Duration, f func()) (stop func() bool) // the timer (afterPause) unless a test sets it + + mu sync.Mutex + retry int // how many unanswered floor reads in a row + since time.Time // when the first of them was asked + pending func() bool } +// floorPauses are the pauses between asks of an unanswered floor read (A-490h), as a node-engine asks a missing trunk build again +// (A-487a): 5 s, 15 s, 30 s, 1, 2 and then 5 minutes; the first answer starts them again. +var floorPauses = []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute} + func (f *versionsFill) kick() { go func() { if err := f.ix.FillVersions(f.ctx, f.ask); err != nil { f.logf("versions: the first fill waits: %v", err) - return - } - // The tag floor, per module the versions hold none of, at every start and every revision (A-490d): the builder's half - // is done once; this half is not. - if f.tags != nil { - if err := f.ix.FillTagFloors(f.ctx, f.tags); err != nil { - f.logf("versions: the tag floor waits: %v", err) - } } }() + go f.run() +} + +// run makes the floor, not waiting for the builder's half: a builder_versions unanswered holds that half for its bound, and the +// floor's own reads answer by themselves. +func (f *versionsFill) run() { + // The floor, per module the versions hold none of, at every start and every revision and again on a growing pause while a + // read is unanswered (A-490d, A-490f, A-490h): the builder's half is done once; this half is not. + if f.floor.Tags == nil && f.floor.Holder == nil { + return + } + err := f.ix.FillTagFloors(f.ctx, f.floor) + f.mu.Lock() + defer f.mu.Unlock() + if err == nil { + f.retry, f.since = 0, time.Time{} + return + } + f.logf("versions: the floor waits: %v", err) + if f.pending != nil || f.ctx.Err() != nil { + return // an ask again is set already + } + pauses := floorPauses + if f.pauses != nil { + pauses = f.pauses + } + pause := pauses[min(f.retry, len(pauses)-1)] + f.retry++ + now := time.Now() + if f.since.IsZero() { + f.since = now + } + // the alarm is five minutes after the first unanswered read: never pause past it before it is raised + if left := f.since.Add(mcc.CallUnansweredAfter).Sub(now); left > 0 && pause > left { + pause = left + } + after := f.after + if after == nil { + after = afterPause + } + f.pending = after(pause, func() { + f.mu.Lock() + f.pending = nil + f.mu.Unlock() + f.run() + }) } // afterRevision runs the fill's kick after each catalog revision kept, once the fill is set. @@ -300,3 +353,27 @@ func tagsVia(call toolCaller) index.TagsReader { } } } + +// holderVia reads the versions a package holder answers for a package: the seat npm-package-registry's or go-package-registry's +// verb versions (A-490g), as a kept call. +func holderVia(call toolCaller) index.HolderReader { + return func(_ context.Context, ecosystem, name string) ([]string, error) { + seat := npmpackageregistry.Seat + if ecosystem == index.EcosystemGo { + seat = gopackageregistry.Seat + } + raw, err := call("seat:"+seat+"."+npmpackageregistry.VerbVersions, npmpackageregistry.VersionsInput{Name: name, Kind: npmpackageregistry.ArtifactKindPackage}) + if err != nil { + return nil, err + } + var a npmpackageregistry.VersionsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s.versions is not a package's versions: %w", seat, err) + } + var out []string + for _, v := range a.Versions { + out = append(out, v.Version) + } + return out, nil + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 1abab54d..0ae08aee 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -174,16 +174,16 @@ type Indexer struct { previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) - tagFilling atomic.Bool - versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) - stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on - fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu + tagFilling atomic.Bool // a FillTagFloors runs (tag_floor.go): one at a time + versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) + stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) + unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) + retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one + retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on + fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu freshMu sync.Mutex recompute sync.Mutex // one Recompute at a time untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) diff --git a/modules/mesh-delivery-indexer/internal/index/tag_floor.go b/modules/mesh-delivery-indexer/internal/index/tag_floor.go index 218a2850..1a9c419f 100644 --- a/modules/mesh-delivery-indexer/internal/index/tag_floor.go +++ b/modules/mesh-delivery-indexer/internal/index/tag_floor.go @@ -2,67 +2,70 @@ package index import ( "context" + "encoding/json" "errors" "fmt" - "slices" "strings" + "time" - "encoding/json" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" ) -// The tag floor of the catalog's versions (design 47 G §412 as A-490c, A-490d and A-490e amend it): for every module the -// state versions holds no version of, the highest tag of the module's Go package in the repository its catalog entry was -// taken in from, read from the seat git's verb tags, protected or not (A-490c), is its first version. It is filled per -// module whenever the indexer starts and at each catalog revision kept, not once (A-490d): the key versions-filled closes -// only the builder's half of the fill. +// The floor of the catalog's versions (design 47 G §412 as A-490c to A-490h amend it): for every module the state versions +// holds no version of, the highest version among every package its catalog entry names is its first version: the Go +// package's tags in the repository the entry was taken in from, read from the seat git's verb tags, protected or not +// (A-490c, A-490e), and what each package holder answers for the package through its seat's verb versions (A-490g). It is +// filled per module whenever the indexer starts and at each catalog revision kept, not once (A-490d, A-490f): the key +// versions-filled closes only the builder's half of the fill. -// TagsCall is the seat verb the tag floor reads, the finding's id when it is unanswered five minutes (§412). -const TagsCall = "git.tags" +// FloorCall is the id of the finding call-unanswered when a floor read (the forge's tags or a package holder's versions) is +// unanswered five minutes after the first (A-490h). +const FloorCall = "mesh-catalog.versions-floor" -var tagsFillSince = "call-since." + keys.EscapeName(TagsCall) +var floorSince = "call-since." + keys.EscapeName(FloorCall) // TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags). type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error) -// tagPrefixesOf is the prefix Go gives the tags of each Go package of the entry in its repository (A-490e): the package's -// directory in the repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root. A -// module with no Go package has none. -func tagPrefixesOf(e meshcatalog.CatalogEntry) []string { - var out []string - for _, a := range e.Artifacts { - if a.Package == nil || a.Package.Ecosystem != "go" { - continue - } - i := strings.Index(a.Package.Name, "/"+e.Repository) - if e.Repository == "" || i < 0 { - continue - } - rest := a.Package.Name[i+1+len(e.Repository):] - var p string - switch { - case rest == "": - p = "v" - case strings.HasPrefix(rest, "/"): - p = rest[1:] + "/v" - default: - continue - } - if !slices.Contains(out, p) { - out = append(out, p) - } - } - return out +// HolderReader reads the versions a package holder answers for a package of an ecosystem (EcosystemGo, EcosystemNpm) +// through its seat's verb versions (A-490g). +type HolderReader func(ctx context.Context, ecosystem, name string) ([]string, error) + +// FloorSources are what the floor reads: the forge's tags and each package holder's versions. +type FloorSources struct { + Tags TagsReader + Holder HolderReader } -// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest tag of its -// Go package. A module whose tags cannot be read is not filled; it is said, call-unanswered after five minutes, and the -// fill is made again at the next start or revision. The floor is a version entry with no artifact and no digest, as the -// builder's numbered-only versions are (A-351), so the builder numbers above it. -func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { - if ix.Versions == nil || ix.Revisions == nil || read == nil { +// tagPrefixOf is the prefix Go gives the tags of a Go package in its repository (A-490e): the package's directory in the +// repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root; none when the package's +// name does not lie in the repository. +func tagPrefixOf(repository string, a meshcatalog.Artifact) (string, bool) { + if a.Package == nil || a.Package.Ecosystem != EcosystemGo || repository == "" { + return "", false + } + i := strings.Index(a.Package.Name, "/"+repository) + if i < 0 { + return "", false + } + switch rest := a.Package.Name[i+1+len(repository):]; { + case rest == "": + return "v", true + case strings.HasPrefix(rest, "/"): + return rest[1:] + "/v", true + } + return "", false +} + +// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest version its +// packages have (A-490g). A read the forge or a holder does not answer fills no module and is returned: the finding +// call-unanswered is written five minutes after the first unanswered read and cleared when a read answers (A-490h); the +// caller asks again on a growing pause. The floor entry holds no artifact and no digest (A-351), so the builder numbers +// above it and a package-published heard later at its version is recorded as its release. +func (ix *Indexer) FillTagFloors(ctx context.Context, src FloorSources) error { + if ix.Versions == nil || ix.Revisions == nil { return errors.New("the state versions or the catalog graph is not kept here") } if !ix.tagFilling.CompareAndSwap(false, true) { @@ -74,14 +77,10 @@ func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { return err } if !found { - return errors.New("the indexer keeps no trunk revision yet: the tag floor waits for its first") + return errors.New("the indexer keeps no trunk revision yet: the floor waits for its first") } var unanswered error for _, e := range entries { - prefixes := tagPrefixesOf(e) - if len(prefixes) == 0 { - continue - } have, err := ModuleVersions(ctx, ix.Versions, e.Module) if err != nil { return err @@ -90,23 +89,40 @@ func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { continue } top := "" - for _, p := range prefixes { - tags, err := read(ctx, e.Repository, p) - if err != nil { - unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err) - top = "" - break - } - for _, t := range tags { - v := strings.TrimPrefix(t, p) - if !strings.HasPrefix(t, p) { + take := func(vs []string, prefix string) { + for _, t := range vs { + if !strings.HasPrefix(t, prefix) { continue } - if _, ok := versionParts(v); ok && (top == "" || compareVersions(v, top) > 0) { + if v := strings.TrimPrefix(t, prefix); versionOK(v) && (top == "" || compareVersions(v, top) > 0) { top = v } } } + for _, a := range e.Artifacts { + if a.Package == nil { + continue + } + if prefix, ok := tagPrefixOf(e.Repository, a); ok && src.Tags != nil { + tags, err := src.Tags(ctx, e.Repository, prefix) + if err != nil { + unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err) + break + } + take(tags, prefix) + } + if (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm) && src.Holder != nil { + vs, err := src.Holder(ctx, a.Package.Ecosystem, a.Package.Name) + if err != nil { + unanswered = fmt.Errorf("the versions of %s %s: %w", a.Package.Ecosystem, a.Package.Name, err) + break + } + take(vs, "") + } + } + if unanswered != nil { + break + } if top == "" { continue } @@ -116,30 +132,33 @@ func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error { return err } if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(e.Module, top), data); err != nil && !errors.Is(err, ErrConflict) { - return fmt.Errorf("the tag floor %s %s is not kept: %w", e.Module, top, err) + return fmt.Errorf("the floor %s %s is not kept: %w", e.Module, top, err) } - ix.logf("versions: %s has none, filled from the forge's tag %s", e.Module, top) + ix.logf("versions: %s has none, filled with %s, the highest of its packages' tags and holders' versions", e.Module, top) } if unanswered != nil { now := ix.now() - since := ix.unansweredSince(ctx, tagsFillSince, now) + since := ix.unansweredSince(ctx, floorSince, now) if ix.Findings != nil && now.Sub(since) >= mcc.CallUnansweredAfter { - f := ix.observed(mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, - fmt.Sprintf("the call %s is unanswered since %s", TagsCall, since.UTC().Format("2006-01-02T15:04:05Z07:00")), + at := since.UTC().Format(time.RFC3339) + f := ix.observed(mcc.ScopeCall, FloorCall, mcc.KindCallUnanswered, + fmt.Sprintf("the floor of the catalog's versions is unanswered since %s", at), "A call whose answer is kept is not answered", - fmt.Sprintf("The tag floor of the catalog's versions reads the forge's tags; they are not read since %s (%v), so a module with none is not filled.", since.UTC().Format("2006-01-02T15:04:05Z07:00"), unanswered), + fmt.Sprintf("The floor of the catalog's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so a module with none is not filled.", at, unanswered), "indexer."+mcc.KindCallUnanswered) if err := ix.Findings.Write(ctx, f); err != nil { - ix.logf("tag floor: the finding %s is not written: %v", TagsCall, err) + ix.logf("floor: the finding %s is not written: %v", FloorCall, err) } } return unanswered } - ix.answered(ctx, tagsFillSince) + ix.answered(ctx, floorSince) if ix.Findings != nil { - if err := ix.clearFinding(ctx, mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, "the forge's tags answer"); err != nil { - ix.logf("tag floor: the finding %s is not cleared: %v", TagsCall, err) + if err := ix.clearFinding(ctx, mcc.ScopeCall, FloorCall, mcc.KindCallUnanswered, "the floor's reads answer"); err != nil { + ix.logf("floor: the finding %s is not cleared: %v", FloorCall, err) } } return nil } + +func versionOK(v string) bool { _, ok := versionParts(v); return ok } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 5c5db15a..574a6fb5 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -120,9 +120,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83) } } - if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { + if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil { return err - } else if found { + } else if found && !isFloor(raw) { ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact) return nil } @@ -212,7 +212,14 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli if err != nil { return err } - if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { + if err := ix.Versions.Create(ctx, key, data); errors.Is(err, ErrConflict) { + // A floor entry (A-490h) holds no artifacts: this publish is that version's release, so it takes the entry's place. + if raw, rev, found, gerr := ix.Versions.Get(ctx, key); gerr == nil && found && isFloor(raw) { + if err := ix.Versions.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) { + return err + } + } + } else if err != nil { return err } if late { @@ -339,3 +346,9 @@ func OutdatedTool(ix *Indexer) stdio.Tool { return ix.Outdated(context.Background(), in) }} } + +// isFloor says whether a kept version is a floor entry (A-490h): it holds no artifact, as no release does. +func isFloor(raw []byte) bool { + var v meshcatalog.VersionEntry + return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0 +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 042917eb..2440b4c1 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -545,7 +545,8 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} entries := map[string]meshcatalog.CatalogEntry{ "mesh-sdk": {Module: "mesh-sdk", Repository: "novox/mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ - {Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}}}, + {Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}, + {Ref: "mesh-sdk:ts", Package: &meshcatalog.ArtifactPackage{Ecosystem: "npm", Name: "@novox/mesh-sdk"}}}}, "root": {Module: "root", Repository: "novox/root", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ {Ref: "root:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/root"}}}}, "held": {Module: "held", Repository: "novox/held", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "held:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/held"}}}}, @@ -576,29 +577,39 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { } return nil, nil } - if err := ix.FillTagFloors(ctx, read); err == nil { + holder := func(_ context.Context, eco, name string) ([]string, error) { + if !up { + return nil, errors.New("no responders") + } + if eco == "npm" && name == "@novox/mesh-sdk" { + return []string{"0.1.18", "0.1.17"}, nil // A-490g: the npm holder is above the Go tags + } + return nil, nil + } + src := FloorSources{Tags: read, Holder: holder} + if err := ix.FillTagFloors(ctx, src); err == nil { t.Fatal("tags not answered filled") } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } now = now.Add(5 * time.Minute) - _ = ix.FillTagFloors(ctx, read) - if h := w.Held(); len(h) != 1 || h[0].ID != TagsCall { + _ = ix.FillTagFloors(ctx, src) + if h := w.Held(); len(h) != 1 || h[0].ID != FloorCall { t.Fatalf("the findings after five minutes: %+v", h) } if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 0 { t.Fatalf("filled while the forge was down: %+v", v) } up = true - if err := ix.FillTagFloors(ctx, read); err != nil { + if err := ix.FillTagFloors(ctx, src); err != nil { t.Fatal(err) } if len(w.Held()) != 0 { t.Errorf("the finding stands after the forge answered: %+v", w.Held()) } - if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.16" { - t.Errorf("mesh-sdk's floor: %+v (the highest of go/v tags, 0.1.16)", v) + if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.18" { + t.Errorf("mesh-sdk's floor: %+v (the highest of its Go tags 0.1.16 and the npm holder's 0.1.18)", v) } if v, _ := ModuleVersions(ctx, ix.Versions, "root"); len(v) != 1 || v[0].Version != "1.2.3" { t.Errorf("a root module's floor: %+v", v) @@ -612,3 +623,36 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { } } } + +// Design 47 G A-490h: a floor entry holds no artifacts, so the package-published heard later at its version is that version's +// release (the entry takes its artifacts and version-released is said), never dropped as released already. +func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC) + var said []meshcatalog.VersionReleasedEvent + ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: newFindings(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, + SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, _ string) error { + said = append(said, e) + return nil + }} + entries := map[string]meshcatalog.CatalogEntry{"mod": {Module: "mod", Repository: "novox/mod", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ + {Ref: "mod:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mod"}}}}} + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + src := FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return []string{"v0.3.0"}, nil }} + if err := ix.FillTagFloors(ctx, src); err != nil { + t.Fatal(err) + } + if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "mod", Artifact: "mod:go", Version: "0.3.0", Digest: "sha256:aa", Branch: "main", Source: 5}); err != nil { + t.Fatal(err) + } + v, _ := ModuleVersions(ctx, ix.Versions, "mod") + if len(v) != 1 || len(v[0].Artifacts) != 1 || v[0].Artifacts[0].Digest != "sha256:aa" || len(said) != 1 { + t.Fatalf("the floor 0.3.0 and its publish: versions %+v, version-released said %d", v, len(said)) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 36c93d78..4b415353 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.tags", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -393,6 +393,10 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR if f == "Subscribe(" && strings.HasSuffix(path, "seatlist.go") { continue } + // The one timer is the growing pause of an unanswered floor read (A-490h), in floorpause.go. + if f == "time.AfterFunc" && strings.HasSuffix(path, "floorpause.go") { + continue + } if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the indexer serves no request subject and holds no timer", path, f) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index da00d9d5..d460a233 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -168,6 +168,8 @@ "invokes": [ "seat:git.trees", "seat:git.tags", + "seat:go-package-registry.versions", + "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", From 69d17f139f4bdd602be95312c7be3ae0b95b6c3e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 17:36:05 +0100 Subject: [PATCH 3/4] Catalog versions: a floor read is a kept call asked until it answers, one module at a time, and the indexer holds no timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A-490i corrects A-490h: the growing-pause timer is gone (floorpause.go and the guard's exemption removed; ADR 0437 §8 stands). Each module's floor is read on its own (FloorModules, FillFloor); a read is a kept call through mesh-sdk's form, asked again at once when the form gives up, so it has no bound; the first unanswered try starts the module's clock and the finding call-unanswered is written five minutes later and cleared on answer. The integration test leaves the forge unserved until a delay, and the fill follows with no restart and no revision. Builds: design 47 G §412, A-490c, A-490d, A-490e, A-490f, A-490g, A-490h, A-490i Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../floor_integration_test.go | 25 +- .../mesh-delivery-indexer/floor_retry_test.go | 43 ---- .../cmd/mesh-delivery-indexer/floorpause.go | 8 - .../cmd/mesh-delivery-indexer/versions.go | 75 ++---- .../internal/index/candidates.go | 2 +- .../internal/index/tag_floor.go | 235 +++++++++++------- .../internal/index/versions_test.go | 50 ++-- .../mesh-delivery-indexer/manifest_test.go | 4 - 8 files changed, 201 insertions(+), 241 deletions(-) delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go index c42a665c..c3c742dc 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go @@ -23,10 +23,10 @@ import ( // seat git and the package holders stood in. A module of the trunk with a Go package and no version in `versions` stays unfilled // while the forge does not answer, and is filled from the highest of its tags and the holder's versions once the forge answers, // on the growing pause (5 s) with no restart and no catalog revision. The finding after five minutes is in the unit tests -// (the five minutes are not waited for here). +// (the five minutes are not waited for here); the kept call asks again by itself, with no restart and no new revision. func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { const pkg = "git.novox.be/novox/mesh-delivery/modules/" + testIndexer - var forgeUp atomic.Bool + var serveTags func() var prefix atomic.Value js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { for _, bucket := range []string{index.Bucket(testIndexer, index.VersionsState), index.Bucket(testIndexer, index.HeardState)} { @@ -58,16 +58,15 @@ func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { t.Fatal(err) } } - serve("mesh.seat.git.tool.tags", func(m *nats.Msg) { - var in git.TagsInput - _ = json.Unmarshal(m.Data, &in) - prefix.Store(in.Prefix) - if !forgeUp.Load() { - reply(m, map[string]any{"error": "the forge does not answer"}) - return - } - reply(m, map[string]any{"result": git.TagsAnswer{Repository: in.Repository, Tags: []string{in.Prefix + "0.4.2", in.Prefix + "0.4.10", "unrelated"}}}) - }) + // the forge answers no one until the test lets it (nobody serves the verb: a transport fault, which the kept call asks again) + serveTags = func() { + serve("mesh.seat.git.tool.tags", func(m *nats.Msg) { + var in git.TagsInput + _ = json.Unmarshal(m.Data, &in) + prefix.Store(in.Prefix) + reply(m, map[string]any{"result": git.TagsAnswer{Repository: in.Repository, Tags: []string{in.Prefix + "0.4.2", in.Prefix + "0.4.10", "unrelated"}}}) + }) + } // the package's name is read from the revision's own files (§676): its go.mod serve("mesh.seat.git.tool.file", func(m *nats.Msg) { var in git.FileInput @@ -90,7 +89,7 @@ func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { if keys, _ := versions.Keys(ctx); len(keys) != 0 { t.Fatalf("versions holds %v while the forge does not answer, want nothing", keys) } - forgeUp.Store(true) + serveTags() var got []string for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { if got, _ = versions.Keys(ctx); len(got) > 0 { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go deleted file mode 100644 index 4f7b28ed..00000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_retry_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "context" - "testing" - "time" - - "mesh-delivery-indexer/internal/index" -) - -// Design 47 G A-490h (as A-487a's backoff): an unanswered floor read is asked again on a growing pause, 5 s, 15 s, 30 s, 1, 2 and -// then 5 minutes (never past the alarm five minutes after the first), until it answers; an answer starts the pauses again. -func TestAnUnansweredFloorReadIsAskedAgainOnAGrowingPause(t *testing.T) { - var pauses []time.Duration - var next func() - f := &versionsFill{ix: &index.Indexer{}, ctx: context.Background(), logf: func(string, ...any) {}, - floor: index.FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return nil, nil }}, - after: func(d time.Duration, fn func()) func() bool { - pauses = append(pauses, d) - next = fn - return func() bool { return true } - }} - f.run() // the indexer keeps no catalog graph: the floor read fails as an unanswered one does - for i := 0; i < 6 && next != nil; i++ { - fn := next - next = nil - fn() - } - want := []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute} - if len(pauses) != 7 { - t.Fatalf("the pauses %v, want 7 asks again", pauses) - } - for i, d := range want { - if pauses[i] != d { - t.Errorf("pause %d is %v, want %v", i, pauses[i], d) - } - } - for _, d := range pauses[5:] { - if d > 5*time.Minute || d < 4*time.Minute { - t.Errorf("a late pause is %v, want the 5-minute pause clamped to the alarm's time", d) - } - } -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go deleted file mode 100644 index c616fb2f..00000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floorpause.go +++ /dev/null @@ -1,8 +0,0 @@ -package main - -import "time" - -// afterPause runs f once after d: the pause between two asks of an unanswered floor read (design 47 G A-490h, as A-487a has a -// node-engine ask again after a growing pause). It is the indexer's one timer, kept in this file alone: the guard -// TestServesNoRequestSubjectAndHoldsNoTicker (ADR 0437 §8) names it. -func afterPause(d time.Duration, f func()) (stop func() bool) { return time.AfterFunc(d, f).Stop } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index 50354565..b9785e96 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "strings" - "sync" "sync/atomic" "time" @@ -14,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" @@ -194,76 +192,41 @@ func versionsVia(call openCaller, inView func(ctx context.Context, module string // versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each // catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next. type versionsFill struct { - ix *index.Indexer - ctx context.Context - ask index.VersionsAsker - floor index.FloorSources // what the floor reads (A-490g); a nil reader fills none - logf func(string, ...any) - pauses []time.Duration // the growing pauses between asks of an unanswered floor read; nil is floorPauses - after func(d time.Duration, f func()) (stop func() bool) // the timer (afterPause) unless a test sets it - - mu sync.Mutex - retry int // how many unanswered floor reads in a row - since time.Time // when the first of them was asked - pending func() bool + ix *index.Indexer + ctx context.Context + ask index.VersionsAsker + floor index.FloorSources // what the floor reads (A-490g); a nil reader fills none + logf func(string, ...any) } -// floorPauses are the pauses between asks of an unanswered floor read (A-490h), as a node-engine asks a missing trunk build again -// (A-487a): 5 s, 15 s, 30 s, 1, 2 and then 5 minutes; the first answer starts them again. -var floorPauses = []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute} - func (f *versionsFill) kick() { go func() { if err := f.ix.FillVersions(f.ctx, f.ask); err != nil { f.logf("versions: the first fill waits: %v", err) } }() - go f.run() + go f.fillFloors() } -// run makes the floor, not waiting for the builder's half: a builder_versions unanswered holds that half for its bound, and the -// floor's own reads answer by themselves. -func (f *versionsFill) run() { - // The floor, per module the versions hold none of, at every start and every revision and again on a growing pause while a - // read is unanswered (A-490d, A-490f, A-490h): the builder's half is done once; this half is not. +// fillFloors fills the floor of each module the versions hold none of, each on its own so that one unanswered read holds no other +// module's (A-490d, A-490f, A-490i): at every start and every revision, not waiting for the builder's half. A read is a kept call +// asked again until it answers, and the indexer holds no timer of its own (ADR 0437 §8). +func (f *versionsFill) fillFloors() { if f.floor.Tags == nil && f.floor.Holder == nil { return } - err := f.ix.FillTagFloors(f.ctx, f.floor) - f.mu.Lock() - defer f.mu.Unlock() - if err == nil { - f.retry, f.since = 0, time.Time{} + modules, err := f.ix.FloorModules(f.ctx) + if err != nil { + f.logf("versions: the floor waits: %v", err) return } - f.logf("versions: the floor waits: %v", err) - if f.pending != nil || f.ctx.Err() != nil { - return // an ask again is set already + for _, m := range modules { + go func() { + if err := f.ix.FillFloor(f.ctx, m, f.floor); err != nil { + f.logf("versions: the floor of %s waits for the next start or revision: %v", m, err) + } + }() } - pauses := floorPauses - if f.pauses != nil { - pauses = f.pauses - } - pause := pauses[min(f.retry, len(pauses)-1)] - f.retry++ - now := time.Now() - if f.since.IsZero() { - f.since = now - } - // the alarm is five minutes after the first unanswered read: never pause past it before it is raised - if left := f.since.Add(mcc.CallUnansweredAfter).Sub(now); left > 0 && pause > left { - pause = left - } - after := f.after - if after == nil { - after = afterPause - } - f.pending = after(pause, func() { - f.mu.Lock() - f.pending = nil - f.mu.Unlock() - f.run() - }) } // afterRevision runs the fill's kick after each catalog revision kept, once the fill is set. diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 0ae08aee..f6ac1594 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -174,7 +174,7 @@ type Indexer struct { previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) - tagFilling atomic.Bool // a FillTagFloors runs (tag_floor.go): one at a time + floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) diff --git a/modules/mesh-delivery-indexer/internal/index/tag_floor.go b/modules/mesh-delivery-indexer/internal/index/tag_floor.go index 1a9c419f..929d93ea 100644 --- a/modules/mesh-delivery-indexer/internal/index/tag_floor.go +++ b/modules/mesh-delivery-indexer/internal/index/tag_floor.go @@ -5,11 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "time" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -24,8 +26,6 @@ import ( // unanswered five minutes after the first (A-490h). const FloorCall = "mesh-catalog.versions-floor" -var floorSince = "call-since." + keys.EscapeName(FloorCall) - // TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags). type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error) @@ -59,106 +59,157 @@ func tagPrefixOf(repository string, a meshcatalog.Artifact) (string, bool) { return "", false } -// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest version its -// packages have (A-490g). A read the forge or a holder does not answer fills no module and is returned: the finding -// call-unanswered is written five minutes after the first unanswered read and cleared when a read answers (A-490h); the -// caller asks again on a growing pause. The floor entry holds no artifact and no digest (A-351), so the builder numbers -// above it and a package-published heard later at its version is recorded as its release. -func (ix *Indexer) FillTagFloors(ctx context.Context, src FloorSources) error { +// FloorModules are the modules of the newest trunk revision the versions hold none of and that name a package a floor can be read +// of; each is filled by its own FillFloor, so one unanswered read holds no other module's floor (A-490i). +func (ix *Indexer) FloorModules(ctx context.Context) ([]string, error) { if ix.Versions == nil || ix.Revisions == nil { - return errors.New("the state versions or the catalog graph is not kept here") + return nil, errors.New("the state versions or the catalog graph is not kept here") } - if !ix.tagFilling.CompareAndSwap(false, true) { + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil { + return nil, err + } + if !found { + return nil, errors.New("the indexer keeps no trunk revision yet: the floor waits for its first") + } + var out []string + for _, e := range entries { + if !slices.ContainsFunc(e.Artifacts, func(a meshcatalog.Artifact) bool { + return a.Package != nil && (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm) + }) { + continue + } + if have, err := ModuleVersions(ctx, ix.Versions, e.Module); err != nil { + return nil, err + } else if len(have) == 0 { + out = append(out, e.Module) + } + } + return out, nil +} + +// FillFloor fills one module's floor, the highest version among its packages' tags and holders' versions (A-490g). Its reads are +// kept calls made through mesh-sdk's form, which asks again through every transport fault; a read it gives up on (its own +// pause spent) is asked again at once, never given up, so the read has no bound (A-490i). The first unanswered try starts the +// module's clock: five minutes later the finding call-unanswered is written, and cleared when a read answers. A read the +// callee answered with a failure ends the fill, left for the next start or revision. One fill of a module runs at a time. +func (ix *Indexer) FillFloor(ctx context.Context, module string, src FloorSources) error { + if _, running := ix.floors.LoadOrStore(module, true); running { return nil } - defer ix.tagFilling.Store(false) + defer ix.floors.Delete(module) + for ctx.Err() == nil { + entry, found, err := ix.newestEntry(ctx, module) + if err != nil || !found { + return err + } + if have, err := ModuleVersions(ctx, ix.Versions, module); err != nil || len(have) > 0 { + return err + } + top, err := ix.floorOf(ctx, entry, src) + if err == nil { + ix.floorAnswered(ctx, module) + return ix.keepFloor(ctx, module, top) + } + if !busretry.Undelivered(err) && !busretry.TimedOut(err) { + return err + } + ix.floorUnanswered(ctx, module, err) + } + return ctx.Err() +} + +func (ix *Indexer) newestEntry(ctx context.Context, module string) (meshcatalog.CatalogEntry, bool, error) { _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found { + return meshcatalog.CatalogEntry{}, false, err + } + for _, e := range entries { + if e.Module == module { + return e, true, nil + } + } + return meshcatalog.CatalogEntry{}, false, nil +} + +// floorOf reads the highest version of the entry's packages; "" when none is held. +func (ix *Indexer) floorOf(ctx context.Context, e meshcatalog.CatalogEntry, src FloorSources) (string, error) { + top := "" + take := func(vs []string, prefix string) { + for _, t := range vs { + if !strings.HasPrefix(t, prefix) { + continue + } + if v := strings.TrimPrefix(t, prefix); versionOK(v) && (top == "" || compareVersions(v, top) > 0) { + top = v + } + } + } + for _, a := range e.Artifacts { + if a.Package == nil { + continue + } + if prefix, ok := tagPrefixOf(e.Repository, a); ok && src.Tags != nil { + tags, err := src.Tags(ctx, e.Repository, prefix) + if err != nil { + return "", fmt.Errorf("the tags of %s: %w", e.Repository, err) + } + take(tags, prefix) + } + if (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm) && src.Holder != nil { + vs, err := src.Holder(ctx, a.Package.Ecosystem, a.Package.Name) + if err != nil { + return "", fmt.Errorf("the versions of %s %s: %w", a.Package.Ecosystem, a.Package.Name, err) + } + take(vs, "") + } + } + return top, nil +} + +func (ix *Indexer) keepFloor(ctx context.Context, module, top string) error { + if top == "" { + return nil + } + now := ix.now().UTC() + data, err := json.Marshal(meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: top, At: &now}) if err != nil { return err } - if !found { - return errors.New("the indexer keeps no trunk revision yet: the floor waits for its first") - } - var unanswered error - for _, e := range entries { - have, err := ModuleVersions(ctx, ix.Versions, e.Module) - if err != nil { - return err - } - if len(have) > 0 { - continue - } - top := "" - take := func(vs []string, prefix string) { - for _, t := range vs { - if !strings.HasPrefix(t, prefix) { - continue - } - if v := strings.TrimPrefix(t, prefix); versionOK(v) && (top == "" || compareVersions(v, top) > 0) { - top = v - } - } - } - for _, a := range e.Artifacts { - if a.Package == nil { - continue - } - if prefix, ok := tagPrefixOf(e.Repository, a); ok && src.Tags != nil { - tags, err := src.Tags(ctx, e.Repository, prefix) - if err != nil { - unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err) - break - } - take(tags, prefix) - } - if (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm) && src.Holder != nil { - vs, err := src.Holder(ctx, a.Package.Ecosystem, a.Package.Name) - if err != nil { - unanswered = fmt.Errorf("the versions of %s %s: %w", a.Package.Ecosystem, a.Package.Name, err) - break - } - take(vs, "") - } - } - if unanswered != nil { - break - } - if top == "" { - continue - } - now := ix.now().UTC() - data, err := json.Marshal(meshcatalog.VersionEntry{V: int64(keys.Version), Module: e.Module, Version: top, At: &now}) - if err != nil { - return err - } - if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(e.Module, top), data); err != nil && !errors.Is(err, ErrConflict) { - return fmt.Errorf("the floor %s %s is not kept: %w", e.Module, top, err) - } - ix.logf("versions: %s has none, filled with %s, the highest of its packages' tags and holders' versions", e.Module, top) - } - if unanswered != nil { - now := ix.now() - since := ix.unansweredSince(ctx, floorSince, now) - if ix.Findings != nil && now.Sub(since) >= mcc.CallUnansweredAfter { - at := since.UTC().Format(time.RFC3339) - f := ix.observed(mcc.ScopeCall, FloorCall, mcc.KindCallUnanswered, - fmt.Sprintf("the floor of the catalog's versions is unanswered since %s", at), - "A call whose answer is kept is not answered", - fmt.Sprintf("The floor of the catalog's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so a module with none is not filled.", at, unanswered), - "indexer."+mcc.KindCallUnanswered) - if err := ix.Findings.Write(ctx, f); err != nil { - ix.logf("floor: the finding %s is not written: %v", FloorCall, err) - } - } - return unanswered - } - ix.answered(ctx, floorSince) - if ix.Findings != nil { - if err := ix.clearFinding(ctx, mcc.ScopeCall, FloorCall, mcc.KindCallUnanswered, "the floor's reads answer"); err != nil { - ix.logf("floor: the finding %s is not cleared: %v", FloorCall, err) - } + if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, top), data); err != nil && !errors.Is(err, ErrConflict) { + return fmt.Errorf("the floor %s %s is not kept: %w", module, top, err) } + ix.logf("versions: %s has none, filled with %s, the highest of its packages' tags and holders' versions", module, top) return nil } +func floorKey(module string) string { return FloorCall + "." + module } + +// floorUnanswered writes call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state). +func (ix *Indexer) floorUnanswered(ctx context.Context, module string, cause error) { + now := ix.now() + since := ix.unansweredSince(ctx, "call-since."+keys.EscapeName(floorKey(module)), now) + if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter { + return + } + at := since.UTC().Format(time.RFC3339) + f := ix.observed(mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered, + fmt.Sprintf("the floor of %s's versions is unanswered since %s", module, at), + "A call whose answer is kept is not answered", + fmt.Sprintf("The floor of %s's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so it has no version.", module, at, cause), + "indexer."+mcc.KindCallUnanswered) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("floor: the finding %s is not written: %v", floorKey(module), err) + } +} + +func (ix *Indexer) floorAnswered(ctx context.Context, module string) { + ix.answered(ctx, "call-since."+keys.EscapeName(floorKey(module))) + if ix.Findings != nil { + if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered, "the floor's reads answer"); err != nil { + ix.logf("floor: the finding %s is not cleared: %v", floorKey(module), err) + } + } +} + func versionOK(v string) bool { _, ok := versionParts(v); return ok } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 2440b4c1..495cfb46 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -562,47 +562,49 @@ func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) { if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("held", "0.2.0"), []byte(`{"v":1,"module":"held","version":"0.2.0"}`)); err != nil { t.Fatal(err) } - up := false + tries := 0 var asked []string read := func(_ context.Context, repo, prefix string) ([]string, error) { asked = append(asked, repo+" "+prefix) - if !up { - return nil, errors.New("no responders") - } - switch repo + " " + prefix { - case "novox/mesh-sdk go/v": + if repo == "novox/mesh-sdk" { + // the forge does not answer the sdk's tags for six minutes, a minute at a time: the fill asks again, never gives up + tries++ + now = now.Add(time.Minute) + if tries == 5 && len(w.Held()) != 0 { + t.Error("the finding is written before five minutes from the first unanswered try") + } + if tries <= 6 { + return nil, errors.New("no responders available for request") + } + if len(w.Held()) != 1 || w.Held()[0].ID != FloorCall+".mesh-sdk" { + t.Errorf("the findings after six minutes: %+v", w.Held()) + } return []string{"go/v0.1.9", "go/v0.1.16", "go/v0.1.12", "go/vnext"}, nil - case "novox/root v": + } + if repo == "novox/root" { return []string{"v1.2.3", "vendor-x"}, nil } return nil, nil } holder := func(_ context.Context, eco, name string) ([]string, error) { - if !up { - return nil, errors.New("no responders") - } if eco == "npm" && name == "@novox/mesh-sdk" { return []string{"0.1.18", "0.1.17"}, nil // A-490g: the npm holder is above the Go tags } return nil, nil } src := FloorSources{Tags: read, Holder: holder} - if err := ix.FillTagFloors(ctx, src); err == nil { - t.Fatal("tags not answered filled") + modules, err := ix.FloorModules(ctx) + if err != nil { + t.Fatal(err) } - if len(w.Held()) != 0 { - t.Fatal("a call unanswered for no time wrote its finding") + if !slices.Equal(modules, []string{"mesh-sdk", "root"}) { + t.Fatalf("the modules with a floor to read: %v (held has a version, image no package)", modules) } - now = now.Add(5 * time.Minute) - _ = ix.FillTagFloors(ctx, src) - if h := w.Held(); len(h) != 1 || h[0].ID != FloorCall { - t.Fatalf("the findings after five minutes: %+v", h) + // each module's floor is read on its own: root is filled whatever the sdk's read waits for + if err := ix.FillFloor(ctx, "root", src); err != nil { + t.Fatal(err) } - if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 0 { - t.Fatalf("filled while the forge was down: %+v", v) - } - up = true - if err := ix.FillTagFloors(ctx, src); err != nil { + if err := ix.FillFloor(ctx, "mesh-sdk", src); err != nil { t.Fatal(err) } if len(w.Held()) != 0 { @@ -645,7 +647,7 @@ func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) { t.Fatal(err) } src := FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return []string{"v0.3.0"}, nil }} - if err := ix.FillTagFloors(ctx, src); err != nil { + if err := ix.FillFloor(ctx, "mod", src); err != nil { t.Fatal(err) } if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "mod", Artifact: "mod:go", Version: "0.3.0", Digest: "sha256:aa", Branch: "main", Source: 5}); err != nil { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 4b415353..d23677fc 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -393,10 +393,6 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR if f == "Subscribe(" && strings.HasSuffix(path, "seatlist.go") { continue } - // The one timer is the growing pause of an unanswered floor read (A-490h), in floorpause.go. - if f == "time.AfterFunc" && strings.HasSuffix(path, "floorpause.go") { - continue - } if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the indexer serves no request subject and holds no timer", path, f) } From b413949ea8420067a3211b45bed55ae29a244a08 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 17:42:56 +0100 Subject: [PATCH 4/4] Catalog versions: the floor's finding clears on every exit of the fill, and the integration test shows the kept call asked again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FillFloor clears the module's call-unanswered finding and its call-since key on every exit but the end of its context (a release gave the module a version, it left the trunk, the callee answered with a failure), as ADR 0668 §3 has: FloorModules never picks the module again. The module's clock starts at its first unanswered try, not when the form gives up. The integration test sets the form's bound to one second, waits for the module's call-since key in the state heard, and only then serves git.tags; it fails with `return err` at the unanswered try. Unit cases for the three exits. Builds: design 47 G §412, A-490g, A-490h, A-490i Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../floor_integration_test.go | 21 ++++++- .../internal/index/tag_floor.go | 25 ++++++-- .../internal/index/versions_test.go | 60 +++++++++++++++++++ 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go index c3c742dc..f64f4522 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/floor_integration_test.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/busretry" "mesh-delivery-indexer/internal/index" ) @@ -22,9 +23,11 @@ import ( // TestIndexerFillsTheFloorOfAModuleWithTags (design 47 G §412 as A-490c to A-490h amend it): the whole indexer on a real bus, the // seat git and the package holders stood in. A module of the trunk with a Go package and no version in `versions` stays unfilled // while the forge does not answer, and is filled from the highest of its tags and the holder's versions once the forge answers, -// on the growing pause (5 s) with no restart and no catalog revision. The finding after five minutes is in the unit tests -// (the five minutes are not waited for here); the kept call asks again by itself, with no restart and no new revision. +// with no restart and no catalog revision: the kept call gives up after its bound (one second here, so that it is spent before the +// forge answers) and the fill asks it again at once (A-490i, ADR 0668 §3). The finding after five minutes is in the unit tests (the +// five minutes are not waited for here). func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { + t.Cleanup(busretry.Waits(time.Second, 50*time.Millisecond, 200*time.Millisecond)) // the form gives up after one second const pkg = "git.novox.be/novox/mesh-delivery/modules/" + testIndexer var serveTags func() var prefix atomic.Value @@ -85,7 +88,19 @@ func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) { if err != nil { t.Fatal(err) } - time.Sleep(2 * time.Second) // the forge is asked at the start and does not answer: nothing is filled + // the forge is asked at the start and does not answer: the form gives up and the module's clock is kept in the state heard + heard, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.HeardState)) + if err != nil { + t.Fatal(err) + } + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if _, err := heard.Get(ctx, index.FloorSinceKey(testIndexer)); err == nil { + break + } + } + if _, err := heard.Get(ctx, index.FloorSinceKey(testIndexer)); err != nil { + t.Fatalf("the unanswered floor read of the module is not kept within 30 s: %v", err) + } if keys, _ := versions.Keys(ctx); len(keys) != 0 { t.Fatalf("versions holds %v while the forge does not answer, want nothing", keys) } diff --git a/modules/mesh-delivery-indexer/internal/index/tag_floor.go b/modules/mesh-delivery-indexer/internal/index/tag_floor.go index 929d93ea..b7e69553 100644 --- a/modules/mesh-delivery-indexer/internal/index/tag_floor.go +++ b/modules/mesh-delivery-indexer/internal/index/tag_floor.go @@ -98,6 +98,14 @@ func (ix *Indexer) FillFloor(ctx context.Context, module string, src FloorSource return nil } defer ix.floors.Delete(module) + // Whatever way the fill ends but by the end of ctx, the module's call is no longer unanswered: it answered, or it left the + // trunk, or a release gave it a version, or the callee answered with a failure (ADR 0668 §3: the finding clears when the call + // is answered), and FloorModules never picks the module again to clear it. + defer func() { + if ctx.Err() == nil { + ix.floorAnswered(ctx, module) + } + }() for ctx.Err() == nil { entry, found, err := ix.newestEntry(ctx, module) if err != nil || !found { @@ -106,15 +114,15 @@ func (ix *Indexer) FillFloor(ctx context.Context, module string, src FloorSource if have, err := ModuleVersions(ctx, ix.Versions, module); err != nil || len(have) > 0 { return err } + first := ix.now() // the clock of the module's unanswered call starts at its try, not when the form gives up top, err := ix.floorOf(ctx, entry, src) if err == nil { - ix.floorAnswered(ctx, module) return ix.keepFloor(ctx, module, top) } if !busretry.Undelivered(err) && !busretry.TimedOut(err) { - return err + return err // answered with a failure: the fill ends, said by the caller } - ix.floorUnanswered(ctx, module, err) + ix.floorUnanswered(ctx, module, first, err) } return ctx.Err() } @@ -185,10 +193,15 @@ func (ix *Indexer) keepFloor(ctx context.Context, module, top string) error { func floorKey(module string) string { return FloorCall + "." + module } +// FloorSinceKey is the key of the state heard that holds when a module's floor call was first unanswered. +func FloorSinceKey(module string) string { return floorSinceKey(module) } + +func floorSinceKey(module string) string { return "call-since." + keys.EscapeName(floorKey(module)) } + // floorUnanswered writes call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state). -func (ix *Indexer) floorUnanswered(ctx context.Context, module string, cause error) { +func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first time.Time, cause error) { now := ix.now() - since := ix.unansweredSince(ctx, "call-since."+keys.EscapeName(floorKey(module)), now) + since := ix.unansweredSince(ctx, floorSinceKey(module), first) if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter { return } @@ -204,7 +217,7 @@ func (ix *Indexer) floorUnanswered(ctx context.Context, module string, cause err } func (ix *Indexer) floorAnswered(ctx context.Context, module string) { - ix.answered(ctx, "call-since."+keys.EscapeName(floorKey(module))) + ix.answered(ctx, floorSinceKey(module)) if ix.Findings != nil { if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered, "the floor's reads answer"); err != nil { ix.logf("floor: the finding %s is not cleared: %v", floorKey(module), err) diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 495cfb46..9a854d85 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -658,3 +658,63 @@ func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) { t.Fatalf("the floor 0.3.0 and its publish: versions %+v, version-released said %d", v, len(said)) } } + +// ADR 0668 §3, design 47 G A-490i: the finding call-unanswered of a module's floor clears, with its call-since key, on every way the +// fill ends but the end of its context: a release gave the module a version, the module left the trunk, or the callee answered with a +// failure (FloorModules never picks the module again to clear it). +func TestTheFloorFindingClearsOnEveryExitOfTheFill(t *testing.T) { + for name, exit := range map[string]func(ctx context.Context, t *testing.T, ix *Indexer) error{ + "a release gave it a version": func(ctx context.Context, t *testing.T, ix *Indexer) error { + if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("mod", "0.5.0"), []byte(`{"v":1,"module":"mod","version":"0.5.0","artifacts":[{"artifact":"mod:go"}]}`)); err != nil { + t.Fatal(err) + } + return errors.New("no responders available for request") + }, + "it left the trunk": func(ctx context.Context, t *testing.T, ix *Indexer) error { + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 2, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{}) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + return errors.New("no responders available for request") + }, + "the callee answered with a failure": func(context.Context, *testing.T, *Indexer) error { return errors.New("the forge refused") }, + } { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC) + w := newFindings() + ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} + entries := map[string]meshcatalog.CatalogEntry{"mod": {Module: "mod", Repository: "novox/mod", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ + {Ref: "mod:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mod"}}}}} + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + tries := 0 + read := func(context.Context, string, string) ([]string, error) { + tries++ + now = now.Add(time.Minute) + if tries <= 6 { + return nil, errors.New("no responders available for request") + } + if len(w.Held()) != 1 { + t.Fatalf("the finding is not written before the exit: %+v", w.Held()) + } + return nil, exit(ctx, t, ix) + } + _ = ix.FillFloor(ctx, "mod", FloorSources{Tags: read}) + if h := w.Held(); len(h) != 0 { + t.Errorf("the finding stands after the fill ended: %+v", h) + } + if _, _, found, _ := ix.Heard.Get(ctx, FloorSinceKey("mod")); found { + t.Error("the call-since key stands after the fill ended") + } + }) + } +}