Catalog versions: fill the tag floor per module with no version, from the forge's Go tags

mesh-catalog.versions answered [] for mesh-sdk although the forge holds go/v0.1.12..v0.1.16: the first fill read only
builder_versions, so a module the builder held nothing of was filled with nothing, and the once-only key closed it. The
indexer now reads, for every module of the newest trunk revision the versions hold none of, every tag of its Go package
from the seat git's verb tags (repository of its catalog entry, prefix the package's directory + "/v", "v" at the root),
fills the highest as its version, at each start and each catalog revision kept, and raises call-unanswered after five
minutes when the forge does not answer.

Builds: design 47 G §412, A-490b, A-490c, A-490d, A-490e
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-19 17:11:15 +01:00
parent 6f8ba0bf69
commit ea4fe471e2
7 changed files with 271 additions and 12 deletions
@@ -491,7 +491,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ix.Registry = registry ix.Registry = registry
named.Changed = ix.RemoveUnlisted named.Changed = ix.RemoveUnlisted
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), logf: logger.Printf} fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), tags: tagsVia(calls.CallKept), logf: logger.Printf}
fillFollow.Store(fill) fillFollow.Store(fill)
fill.kick() fill.kick()
ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) {
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings"
"sync/atomic" "sync/atomic"
"time" "time"
@@ -12,10 +13,12 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index" "mesh-delivery-indexer/internal/index"
@@ -190,6 +193,7 @@ type versionsFill struct {
ix *index.Indexer ix *index.Indexer
ctx context.Context ctx context.Context
ask index.VersionsAsker ask index.VersionsAsker
tags index.TagsReader // the tag floor's reader (A-490d); nil fills none
logf func(string, ...any) logf func(string, ...any)
} }
@@ -197,6 +201,14 @@ func (f *versionsFill) kick() {
go func() { go func() {
if err := f.ix.FillVersions(f.ctx, f.ask); err != nil { if err := f.ix.FillVersions(f.ctx, f.ask); err != nil {
f.logf("versions: the first fill waits: %v", err) f.logf("versions: the first fill waits: %v", err)
return
}
// The tag floor, per module the versions hold none of, at every start and every revision (A-490d): the builder's half
// is done once; this half is not.
if f.tags != nil {
if err := f.ix.FillTagFloors(f.ctx, f.tags); err != nil {
f.logf("versions: the tag floor waits: %v", err)
}
} }
}() }()
} }
@@ -266,3 +278,25 @@ func runningWholeVia(js jetstream.JetStream) index.RunningReader {
} }
} }
} }
// tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by
// page until none is more (design 47 G §412, A-490c: protected or not).
func tagsVia(call toolCaller) index.TagsReader {
return func(_ context.Context, repository, prefix string) ([]string, error) {
var out []string
for page := int64(1); ; page++ {
var a git.TagsAnswer
in := git.TagsInput{Repository: trunk.RepositoryName(repository), Prefix: prefix, Page: page}
if !strings.Contains(in.Repository, "/") {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
if err := askGit(call, git.VerbTags, in, &a); err != nil {
return nil, err
}
out = append(out, a.Tags...)
if !a.More {
return out, nil
}
}
}
}
@@ -174,15 +174,16 @@ type Indexer struct {
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) tagFilling atomic.Bool
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
freshMu sync.Mutex freshMu sync.Mutex
recompute sync.Mutex // one Recompute at a time recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
@@ -0,0 +1,145 @@
package index
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"encoding/json"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The tag floor of the catalog's versions (design 47 G §412 as A-490c, A-490d and A-490e amend it): for every module the
// state versions holds no version of, the highest tag of the module's Go package in the repository its catalog entry was
// taken in from, read from the seat git's verb tags, protected or not (A-490c), is its first version. It is filled per
// module whenever the indexer starts and at each catalog revision kept, not once (A-490d): the key versions-filled closes
// only the builder's half of the fill.
// TagsCall is the seat verb the tag floor reads, the finding's id when it is unanswered five minutes (§412).
const TagsCall = "git.tags"
var tagsFillSince = "call-since." + keys.EscapeName(TagsCall)
// TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags).
type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error)
// tagPrefixesOf is the prefix Go gives the tags of each Go package of the entry in its repository (A-490e): the package's
// directory in the repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root. A
// module with no Go package has none.
func tagPrefixesOf(e meshcatalog.CatalogEntry) []string {
var out []string
for _, a := range e.Artifacts {
if a.Package == nil || a.Package.Ecosystem != "go" {
continue
}
i := strings.Index(a.Package.Name, "/"+e.Repository)
if e.Repository == "" || i < 0 {
continue
}
rest := a.Package.Name[i+1+len(e.Repository):]
var p string
switch {
case rest == "":
p = "v"
case strings.HasPrefix(rest, "/"):
p = rest[1:] + "/v"
default:
continue
}
if !slices.Contains(out, p) {
out = append(out, p)
}
}
return out
}
// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest tag of its
// Go package. A module whose tags cannot be read is not filled; it is said, call-unanswered after five minutes, and the
// fill is made again at the next start or revision. The floor is a version entry with no artifact and no digest, as the
// builder's numbered-only versions are (A-351), so the builder numbers above it.
func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error {
if ix.Versions == nil || ix.Revisions == nil || read == nil {
return errors.New("the state versions or the catalog graph is not kept here")
}
if !ix.tagFilling.CompareAndSwap(false, true) {
return nil
}
defer ix.tagFilling.Store(false)
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return err
}
if !found {
return errors.New("the indexer keeps no trunk revision yet: the tag floor waits for its first")
}
var unanswered error
for _, e := range entries {
prefixes := tagPrefixesOf(e)
if len(prefixes) == 0 {
continue
}
have, err := ModuleVersions(ctx, ix.Versions, e.Module)
if err != nil {
return err
}
if len(have) > 0 {
continue
}
top := ""
for _, p := range prefixes {
tags, err := read(ctx, e.Repository, p)
if err != nil {
unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err)
top = ""
break
}
for _, t := range tags {
v := strings.TrimPrefix(t, p)
if !strings.HasPrefix(t, p) {
continue
}
if _, ok := versionParts(v); ok && (top == "" || compareVersions(v, top) > 0) {
top = v
}
}
}
if top == "" {
continue
}
now := ix.now().UTC()
data, err := json.Marshal(meshcatalog.VersionEntry{V: int64(keys.Version), Module: e.Module, Version: top, At: &now})
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(e.Module, top), data); err != nil && !errors.Is(err, ErrConflict) {
return fmt.Errorf("the tag floor %s %s is not kept: %w", e.Module, top, err)
}
ix.logf("versions: %s has none, filled from the forge's tag %s", e.Module, top)
}
if unanswered != nil {
now := ix.now()
since := ix.unansweredSince(ctx, tagsFillSince, now)
if ix.Findings != nil && now.Sub(since) >= mcc.CallUnansweredAfter {
f := ix.observed(mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered,
fmt.Sprintf("the call %s is unanswered since %s", TagsCall, since.UTC().Format("2006-01-02T15:04:05Z07:00")),
"A call whose answer is kept is not answered",
fmt.Sprintf("The tag floor of the catalog's versions reads the forge's tags; they are not read since %s (%v), so a module with none is not filled.", since.UTC().Format("2006-01-02T15:04:05Z07:00"), unanswered),
"indexer."+mcc.KindCallUnanswered)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("tag floor: the finding %s is not written: %v", TagsCall, err)
}
}
return unanswered
}
ix.answered(ctx, tagsFillSince)
if ix.Findings != nil {
if err := ix.clearFinding(ctx, mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, "the forge's tags answer"); err != nil {
ix.logf("tag floor: the finding %s is not cleared: %v", TagsCall, err)
}
}
return nil
}
@@ -534,3 +534,81 @@ func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err) t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err)
} }
} }
// Design 47 G §412 as A-490c, A-490d and A-490e amend it: a module the versions hold none of is filled from the highest tag of its
// Go package in the repository it was taken in from, per module and not once; a forge that does not answer fills nothing and
// raises call-unanswered after five minutes.
func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
w := newFindings()
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }}
entries := map[string]meshcatalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk", Repository: "novox/mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
{Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}}},
"root": {Module: "root", Repository: "novox/root", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
{Ref: "root:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/root"}}}},
"held": {Module: "held", Repository: "novox/held", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "held:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/held"}}}},
"image": {Module: "image", Repository: "novox/image", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "image:image", Kind: "image"}}},
}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("held", "0.2.0"), []byte(`{"v":1,"module":"held","version":"0.2.0"}`)); err != nil {
t.Fatal(err)
}
up := false
var asked []string
read := func(_ context.Context, repo, prefix string) ([]string, error) {
asked = append(asked, repo+" "+prefix)
if !up {
return nil, errors.New("no responders")
}
switch repo + " " + prefix {
case "novox/mesh-sdk go/v":
return []string{"go/v0.1.9", "go/v0.1.16", "go/v0.1.12", "go/vnext"}, nil
case "novox/root v":
return []string{"v1.2.3", "vendor-x"}, nil
}
return nil, nil
}
if err := ix.FillTagFloors(ctx, read); err == nil {
t.Fatal("tags not answered filled")
}
if len(w.Held()) != 0 {
t.Fatal("a call unanswered for no time wrote its finding")
}
now = now.Add(5 * time.Minute)
_ = ix.FillTagFloors(ctx, read)
if h := w.Held(); len(h) != 1 || h[0].ID != TagsCall {
t.Fatalf("the findings after five minutes: %+v", h)
}
if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 0 {
t.Fatalf("filled while the forge was down: %+v", v)
}
up = true
if err := ix.FillTagFloors(ctx, read); err != nil {
t.Fatal(err)
}
if len(w.Held()) != 0 {
t.Errorf("the finding stands after the forge answered: %+v", w.Held())
}
if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.16" {
t.Errorf("mesh-sdk's floor: %+v (the highest of go/v tags, 0.1.16)", v)
}
if v, _ := ModuleVersions(ctx, ix.Versions, "root"); len(v) != 1 || v[0].Version != "1.2.3" {
t.Errorf("a root module's floor: %+v", v)
}
if v, _ := ModuleVersions(ctx, ix.Versions, "held"); len(v) != 1 || v[0].Version != "0.2.0" {
t.Errorf("a module with a version is filled: %+v", v)
}
for _, a := range asked {
if strings.HasPrefix(a, "novox/held") || strings.HasPrefix(a, "novox/image") {
t.Errorf("tags read for a module with a version or no Go package: %s", a)
}
}
}
@@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900). // commit is in another (issue 1900).
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", if want := []string{"seat:git.trees", "seat:git.tags", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want) t.Errorf("invokes = %v, want %v", m.Invokes, want)
} }
@@ -416,7 +416,7 @@ func TestEveryGiteaToolTheCodeCallsIsInInvokes(t *testing.T) {
if err != nil || len(files) == 0 { if err != nil || len(files) == 0 {
t.Fatalf("no program files: %v", err) t.Fatalf("no program files: %v", err)
} }
verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch} verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch, "VerbTags": git.VerbTags}
forge, asked := regexp.MustCompile(`gitea\.gitea_[a-z_]+`), regexp.MustCompile(`git\.Verb[A-Za-z]+`) forge, asked := regexp.MustCompile(`gitea\.gitea_[a-z_]+`), regexp.MustCompile(`git\.Verb[A-Za-z]+`)
seen := 0 seen := 0
for _, f := range files { for _, f := range files {
@@ -167,6 +167,7 @@
], ],
"invokes": [ "invokes": [
"seat:git.trees", "seat:git.trees",
"seat:git.tags",
"seat:git.file", "seat:git.file",
"seat:git.contains", "seat:git.contains",
"seat:git.open-pulls", "seat:git.open-pulls",