Catalog versions: each module's version floor from its Go tags and its package holders, per module, as a kept call (A-490c..A-490i, §412) #260
@@ -491,7 +491,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ix.Registry = registry
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
|
||||
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), logf: logger.Printf}
|
||||
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), tags: tagsVia(calls.CallKept), logf: logger.Printf}
|
||||
fillFollow.Store(fill)
|
||||
fill.kick()
|
||||
ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -12,10 +13,12 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -190,6 +193,7 @@ type versionsFill struct {
|
||||
ix *index.Indexer
|
||||
ctx context.Context
|
||||
ask index.VersionsAsker
|
||||
tags index.TagsReader // the tag floor's reader (A-490d); nil fills none
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
@@ -197,6 +201,14 @@ func (f *versionsFill) kick() {
|
||||
go func() {
|
||||
if err := f.ix.FillVersions(f.ctx, f.ask); err != nil {
|
||||
f.logf("versions: the first fill waits: %v", err)
|
||||
return
|
||||
}
|
||||
// The tag floor, per module the versions hold none of, at every start and every revision (A-490d): the builder's half
|
||||
// is done once; this half is not.
|
||||
if f.tags != nil {
|
||||
if err := f.ix.FillTagFloors(f.ctx, f.tags); err != nil {
|
||||
f.logf("versions: the tag floor waits: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -266,3 +278,25 @@ func runningWholeVia(js jetstream.JetStream) index.RunningReader {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by
|
||||
// page until none is more (design 47 G §412, A-490c: protected or not).
|
||||
func tagsVia(call toolCaller) index.TagsReader {
|
||||
return func(_ context.Context, repository, prefix string) ([]string, error) {
|
||||
var out []string
|
||||
for page := int64(1); ; page++ {
|
||||
var a git.TagsAnswer
|
||||
in := git.TagsInput{Repository: trunk.RepositoryName(repository), Prefix: prefix, Page: page}
|
||||
if !strings.Contains(in.Repository, "/") {
|
||||
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
if err := askGit(call, git.VerbTags, in, &a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a.Tags...)
|
||||
if !a.More {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,15 +174,16 @@ type Indexer struct {
|
||||
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
tagFilling atomic.Bool
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
freshMu sync.Mutex
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"encoding/json"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The tag floor of the catalog's versions (design 47 G §412 as A-490c, A-490d and A-490e amend it): for every module the
|
||||
// state versions holds no version of, the highest tag of the module's Go package in the repository its catalog entry was
|
||||
// taken in from, read from the seat git's verb tags, protected or not (A-490c), is its first version. It is filled per
|
||||
// module whenever the indexer starts and at each catalog revision kept, not once (A-490d): the key versions-filled closes
|
||||
// only the builder's half of the fill.
|
||||
|
||||
// TagsCall is the seat verb the tag floor reads, the finding's id when it is unanswered five minutes (§412).
|
||||
const TagsCall = "git.tags"
|
||||
|
||||
var tagsFillSince = "call-since." + keys.EscapeName(TagsCall)
|
||||
|
||||
// TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags).
|
||||
type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error)
|
||||
|
||||
// tagPrefixesOf is the prefix Go gives the tags of each Go package of the entry in its repository (A-490e): the package's
|
||||
// directory in the repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root. A
|
||||
// module with no Go package has none.
|
||||
func tagPrefixesOf(e meshcatalog.CatalogEntry) []string {
|
||||
var out []string
|
||||
for _, a := range e.Artifacts {
|
||||
if a.Package == nil || a.Package.Ecosystem != "go" {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(a.Package.Name, "/"+e.Repository)
|
||||
if e.Repository == "" || i < 0 {
|
||||
continue
|
||||
}
|
||||
rest := a.Package.Name[i+1+len(e.Repository):]
|
||||
var p string
|
||||
switch {
|
||||
case rest == "":
|
||||
p = "v"
|
||||
case strings.HasPrefix(rest, "/"):
|
||||
p = rest[1:] + "/v"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(out, p) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FillTagFloors fills the version of every module of the newest trunk revision that holds none from the highest tag of its
|
||||
// Go package. A module whose tags cannot be read is not filled; it is said, call-unanswered after five minutes, and the
|
||||
// fill is made again at the next start or revision. The floor is a version entry with no artifact and no digest, as the
|
||||
// builder's numbered-only versions are (A-351), so the builder numbers above it.
|
||||
func (ix *Indexer) FillTagFloors(ctx context.Context, read TagsReader) error {
|
||||
if ix.Versions == nil || ix.Revisions == nil || read == nil {
|
||||
return errors.New("the state versions or the catalog graph is not kept here")
|
||||
}
|
||||
if !ix.tagFilling.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
defer ix.tagFilling.Store(false)
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return errors.New("the indexer keeps no trunk revision yet: the tag floor waits for its first")
|
||||
}
|
||||
var unanswered error
|
||||
for _, e := range entries {
|
||||
prefixes := tagPrefixesOf(e)
|
||||
if len(prefixes) == 0 {
|
||||
continue
|
||||
}
|
||||
have, err := ModuleVersions(ctx, ix.Versions, e.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(have) > 0 {
|
||||
continue
|
||||
}
|
||||
top := ""
|
||||
for _, p := range prefixes {
|
||||
tags, err := read(ctx, e.Repository, p)
|
||||
if err != nil {
|
||||
unanswered = fmt.Errorf("the tags of %s: %w", e.Repository, err)
|
||||
top = ""
|
||||
break
|
||||
}
|
||||
for _, t := range tags {
|
||||
v := strings.TrimPrefix(t, p)
|
||||
if !strings.HasPrefix(t, p) {
|
||||
continue
|
||||
}
|
||||
if _, ok := versionParts(v); ok && (top == "" || compareVersions(v, top) > 0) {
|
||||
top = v
|
||||
}
|
||||
}
|
||||
}
|
||||
if top == "" {
|
||||
continue
|
||||
}
|
||||
now := ix.now().UTC()
|
||||
data, err := json.Marshal(meshcatalog.VersionEntry{V: int64(keys.Version), Module: e.Module, Version: top, At: &now})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(e.Module, top), data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return fmt.Errorf("the tag floor %s %s is not kept: %w", e.Module, top, err)
|
||||
}
|
||||
ix.logf("versions: %s has none, filled from the forge's tag %s", e.Module, top)
|
||||
}
|
||||
if unanswered != nil {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, tagsFillSince, now)
|
||||
if ix.Findings != nil && now.Sub(since) >= mcc.CallUnansweredAfter {
|
||||
f := ix.observed(mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", TagsCall, since.UTC().Format("2006-01-02T15:04:05Z07:00")),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The tag floor of the catalog's versions reads the forge's tags; they are not read since %s (%v), so a module with none is not filled.", since.UTC().Format("2006-01-02T15:04:05Z07:00"), unanswered),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("tag floor: the finding %s is not written: %v", TagsCall, err)
|
||||
}
|
||||
}
|
||||
return unanswered
|
||||
}
|
||||
ix.answered(ctx, tagsFillSince)
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, TagsCall, mcc.KindCallUnanswered, "the forge's tags answer"); err != nil {
|
||||
ix.logf("tag floor: the finding %s is not cleared: %v", TagsCall, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -534,3 +534,81 @@ func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
|
||||
t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §412 as A-490c, A-490d and A-490e amend it: a module the versions hold none of is filled from the highest tag of its
|
||||
// Go package in the repository it was taken in from, per module and not once; a forge that does not answer fills nothing and
|
||||
// raises call-unanswered after five minutes.
|
||||
func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
|
||||
w := newFindings()
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"mesh-sdk": {Module: "mesh-sdk", Repository: "novox/mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}}},
|
||||
"root": {Module: "root", Repository: "novox/root", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "root:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/root"}}}},
|
||||
"held": {Module: "held", Repository: "novox/held", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "held:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/held"}}}},
|
||||
"image": {Module: "image", Repository: "novox/image", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "image:image", Kind: "image"}}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("held", "0.2.0"), []byte(`{"v":1,"module":"held","version":"0.2.0"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
up := false
|
||||
var asked []string
|
||||
read := func(_ context.Context, repo, prefix string) ([]string, error) {
|
||||
asked = append(asked, repo+" "+prefix)
|
||||
if !up {
|
||||
return nil, errors.New("no responders")
|
||||
}
|
||||
switch repo + " " + prefix {
|
||||
case "novox/mesh-sdk go/v":
|
||||
return []string{"go/v0.1.9", "go/v0.1.16", "go/v0.1.12", "go/vnext"}, nil
|
||||
case "novox/root v":
|
||||
return []string{"v1.2.3", "vendor-x"}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if err := ix.FillTagFloors(ctx, read); err == nil {
|
||||
t.Fatal("tags not answered filled")
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Fatal("a call unanswered for no time wrote its finding")
|
||||
}
|
||||
now = now.Add(5 * time.Minute)
|
||||
_ = ix.FillTagFloors(ctx, read)
|
||||
if h := w.Held(); len(h) != 1 || h[0].ID != TagsCall {
|
||||
t.Fatalf("the findings after five minutes: %+v", h)
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 0 {
|
||||
t.Fatalf("filled while the forge was down: %+v", v)
|
||||
}
|
||||
up = true
|
||||
if err := ix.FillTagFloors(ctx, read); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Errorf("the finding stands after the forge answered: %+v", w.Held())
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.16" {
|
||||
t.Errorf("mesh-sdk's floor: %+v (the highest of go/v tags, 0.1.16)", v)
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "root"); len(v) != 1 || v[0].Version != "1.2.3" {
|
||||
t.Errorf("a root module's floor: %+v", v)
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "held"); len(v) != 1 || v[0].Version != "0.2.0" {
|
||||
t.Errorf("a module with a version is filled: %+v", v)
|
||||
}
|
||||
for _, a := range asked {
|
||||
if strings.HasPrefix(a, "novox/held") || strings.HasPrefix(a, "novox/image") {
|
||||
t.Errorf("tags read for a module with a version or no Go package: %s", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
@@ -416,7 +416,7 @@ func TestEveryGiteaToolTheCodeCallsIsInInvokes(t *testing.T) {
|
||||
if err != nil || len(files) == 0 {
|
||||
t.Fatalf("no program files: %v", err)
|
||||
}
|
||||
verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch}
|
||||
verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch, "VerbTags": git.VerbTags}
|
||||
forge, asked := regexp.MustCompile(`gitea\.gitea_[a-z_]+`), regexp.MustCompile(`git\.Verb[A-Za-z]+`)
|
||||
seen := 0
|
||||
for _, f := range files {
|
||||
|
||||
@@ -167,6 +167,7 @@
|
||||
],
|
||||
"invokes": [
|
||||
"seat:git.trees",
|
||||
"seat:git.tags",
|
||||
"seat:git.file",
|
||||
"seat:git.contains",
|
||||
"seat:git.open-pulls",
|
||||
|
||||
Reference in New Issue
Block a user