Merge pull request 'Catalog versions: each module's version floor from its Go tags and its package holders, per module, as a kept call (A-490c..A-490i, §412)' (#260) from fix/catalog-versions-first-fill into main
This commit was merged in pull request #260.
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// TestIndexerFillsTheFloorOfAModuleWithTags (design 47 G §412 as A-490c to A-490h amend it): the whole indexer on a real bus, the
|
||||
// seat git and the package holders stood in. A module of the trunk with a Go package and no version in `versions` stays unfilled
|
||||
// while the forge does not answer, and is filled from the highest of its tags and the holder's versions once the forge answers,
|
||||
// with no restart and no catalog revision: the kept call gives up after its bound (one second here, so that it is spent before the
|
||||
// forge answers) and the fill asks it again at once (A-490i, ADR 0668 §3). The finding after five minutes is in the unit tests (the
|
||||
// five minutes are not waited for here).
|
||||
func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) {
|
||||
t.Cleanup(busretry.Waits(time.Second, 50*time.Millisecond, 200*time.Millisecond)) // the form gives up after one second
|
||||
const pkg = "git.novox.be/novox/mesh-delivery/modules/" + testIndexer
|
||||
var serveTags func()
|
||||
var prefix atomic.Value
|
||||
js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
for _, bucket := range []string{index.Bucket(testIndexer, index.VersionsState), index.Bucket(testIndexer, index.HeardState)} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// the module's artifacts are Go packages here, so that the indexer's own entries carry their names (read from go.mod, §676)
|
||||
e := trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte {
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go", "kind": "package", "language": "go"}}}
|
||||
out, _ := json.Marshal(m)
|
||||
return out
|
||||
})
|
||||
for i := range e.Artifacts {
|
||||
e.Artifacts[i].Package = &meshcatalog.ArtifactPackage{Ecosystem: index.EcosystemGo, Name: pkg}
|
||||
}
|
||||
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: e})
|
||||
reply := func(m *nats.Msg, body map[string]any) {
|
||||
body["node"] = "n0"
|
||||
raw, _ := json.Marshal(body)
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
serve := func(subject string, h func(m *nats.Msg)) {
|
||||
if _, err := js.Conn().Subscribe(subject, h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// the forge answers no one until the test lets it (nobody serves the verb: a transport fault, which the kept call asks again)
|
||||
serveTags = func() {
|
||||
serve("mesh.seat.git.tool.tags", func(m *nats.Msg) {
|
||||
var in git.TagsInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
prefix.Store(in.Prefix)
|
||||
reply(m, map[string]any{"result": git.TagsAnswer{Repository: in.Repository, Tags: []string{in.Prefix + "0.4.2", in.Prefix + "0.4.10", "unrelated"}}})
|
||||
})
|
||||
}
|
||||
// the package's name is read from the revision's own files (§676): its go.mod
|
||||
serve("mesh.seat.git.tool.file", func(m *nats.Msg) {
|
||||
var in git.FileInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
if strings.HasSuffix(in.Path, "go.mod") {
|
||||
reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Content: "module " + pkg + "\n\ngo 1.26\n"}})
|
||||
return
|
||||
}
|
||||
reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Absent: true}})
|
||||
})
|
||||
serve("mesh.seat.go-package-registry.tool.versions", func(m *nats.Msg) {
|
||||
reply(m, map[string]any{"result": map[string]any{"name": pkg, "versions": []any{}}})
|
||||
})
|
||||
})
|
||||
versions, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.VersionsState))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// the forge is asked at the start and does not answer: the form gives up and the module's clock is kept in the state heard
|
||||
heard, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.HeardState))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if _, err := heard.Get(ctx, index.FloorSinceKey(testIndexer)); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, err := heard.Get(ctx, index.FloorSinceKey(testIndexer)); err != nil {
|
||||
t.Fatalf("the unanswered floor read of the module is not kept within 30 s: %v", err)
|
||||
}
|
||||
if keys, _ := versions.Keys(ctx); len(keys) != 0 {
|
||||
t.Fatalf("versions holds %v while the forge does not answer, want nothing", keys)
|
||||
}
|
||||
serveTags()
|
||||
var got []string
|
||||
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if got, _ = versions.Keys(ctx); len(got) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(got) != 1 || !strings.Contains(got[0], "0.4.10") {
|
||||
t.Fatalf("versions holds %v within 30 s of the forge answering, want the floor 0.4.10 of its tags", got)
|
||||
}
|
||||
if p, _ := prefix.Load().(string); p != "modules/"+testIndexer+"/v" {
|
||||
t.Errorf("the tags were read with the prefix %q, want the package's directory and /v", p)
|
||||
}
|
||||
}
|
||||
@@ -392,11 +392,17 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
|
||||
// trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit.
|
||||
func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream) meshcatalog.CatalogEntry {
|
||||
return trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte { return raw })
|
||||
}
|
||||
|
||||
// trunkIndexerEntryOf is trunkIndexerEntry with the manifest's bytes changed by edit first.
|
||||
func trunkIndexerEntryOf(t *testing.T, ctx context.Context, js jetstream.JetStream, edit func([]byte) []byte) meshcatalog.CatalogEntry {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw = edit(raw)
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -491,7 +491,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ix.Registry = registry
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
|
||||
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), logf: logger.Printf}
|
||||
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), floor: index.FloorSources{Tags: tagsVia(calls.CallKept), Holder: holderVia(calls.CallKept)}, logf: logger.Printf}
|
||||
fillFollow.Store(fill)
|
||||
fill.kick()
|
||||
ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -12,10 +13,14 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -187,10 +192,11 @@ func versionsVia(call openCaller, inView func(ctx context.Context, module string
|
||||
// versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each
|
||||
// catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next.
|
||||
type versionsFill struct {
|
||||
ix *index.Indexer
|
||||
ctx context.Context
|
||||
ask index.VersionsAsker
|
||||
logf func(string, ...any)
|
||||
ix *index.Indexer
|
||||
ctx context.Context
|
||||
ask index.VersionsAsker
|
||||
floor index.FloorSources // what the floor reads (A-490g); a nil reader fills none
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
func (f *versionsFill) kick() {
|
||||
@@ -199,6 +205,28 @@ func (f *versionsFill) kick() {
|
||||
f.logf("versions: the first fill waits: %v", err)
|
||||
}
|
||||
}()
|
||||
go f.fillFloors()
|
||||
}
|
||||
|
||||
// fillFloors fills the floor of each module the versions hold none of, each on its own so that one unanswered read holds no other
|
||||
// module's (A-490d, A-490f, A-490i): at every start and every revision, not waiting for the builder's half. A read is a kept call
|
||||
// asked again until it answers, and the indexer holds no timer of its own (ADR 0437 §8).
|
||||
func (f *versionsFill) fillFloors() {
|
||||
if f.floor.Tags == nil && f.floor.Holder == nil {
|
||||
return
|
||||
}
|
||||
modules, err := f.ix.FloorModules(f.ctx)
|
||||
if err != nil {
|
||||
f.logf("versions: the floor waits: %v", err)
|
||||
return
|
||||
}
|
||||
for _, m := range modules {
|
||||
go func() {
|
||||
if err := f.ix.FillFloor(f.ctx, m, f.floor); err != nil {
|
||||
f.logf("versions: the floor of %s waits for the next start or revision: %v", m, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// afterRevision runs the fill's kick after each catalog revision kept, once the fill is set.
|
||||
@@ -266,3 +294,49 @@ func runningWholeVia(js jetstream.JetStream) index.RunningReader {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by
|
||||
// page until none is more (design 47 G §412, A-490c: protected or not).
|
||||
func tagsVia(call toolCaller) index.TagsReader {
|
||||
return func(_ context.Context, repository, prefix string) ([]string, error) {
|
||||
var out []string
|
||||
for page := int64(1); ; page++ {
|
||||
var a git.TagsAnswer
|
||||
in := git.TagsInput{Repository: trunk.RepositoryName(repository), Prefix: prefix, Page: page}
|
||||
if !strings.Contains(in.Repository, "/") {
|
||||
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
if err := askGit(call, git.VerbTags, in, &a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a.Tags...)
|
||||
if !a.More {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// holderVia reads the versions a package holder answers for a package: the seat npm-package-registry's or go-package-registry's
|
||||
// verb versions (A-490g), as a kept call.
|
||||
func holderVia(call toolCaller) index.HolderReader {
|
||||
return func(_ context.Context, ecosystem, name string) ([]string, error) {
|
||||
seat := npmpackageregistry.Seat
|
||||
if ecosystem == index.EcosystemGo {
|
||||
seat = gopackageregistry.Seat
|
||||
}
|
||||
raw, err := call("seat:"+seat+"."+npmpackageregistry.VerbVersions, npmpackageregistry.VersionsInput{Name: name, Kind: npmpackageregistry.ArtifactKindPackage})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a npmpackageregistry.VersionsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s.versions is not a package's versions: %w", seat, err)
|
||||
}
|
||||
var out []string
|
||||
for _, v := range a.Versions {
|
||||
out = append(out, v.Version)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,6 +174,7 @@ type Indexer struct {
|
||||
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The floor of the catalog's versions (design 47 G §412 as A-490c to A-490h amend it): for every module the state versions
|
||||
// holds no version of, the highest version among every package its catalog entry names is its first version: the Go
|
||||
// package's tags in the repository the entry was taken in from, read from the seat git's verb tags, protected or not
|
||||
// (A-490c, A-490e), and what each package holder answers for the package through its seat's verb versions (A-490g). It is
|
||||
// filled per module whenever the indexer starts and at each catalog revision kept, not once (A-490d, A-490f): the key
|
||||
// versions-filled closes only the builder's half of the fill.
|
||||
|
||||
// FloorCall is the id of the finding call-unanswered when a floor read (the forge's tags or a package holder's versions) is
|
||||
// unanswered five minutes after the first (A-490h).
|
||||
const FloorCall = "mesh-catalog.versions-floor"
|
||||
|
||||
// TagsReader reads every tag of a repository that starts with prefix, all pages (the seat git's verb tags).
|
||||
type TagsReader func(ctx context.Context, repository, prefix string) ([]string, error)
|
||||
|
||||
// HolderReader reads the versions a package holder answers for a package of an ecosystem (EcosystemGo, EcosystemNpm)
|
||||
// through its seat's verb versions (A-490g).
|
||||
type HolderReader func(ctx context.Context, ecosystem, name string) ([]string, error)
|
||||
|
||||
// FloorSources are what the floor reads: the forge's tags and each package holder's versions.
|
||||
type FloorSources struct {
|
||||
Tags TagsReader
|
||||
Holder HolderReader
|
||||
}
|
||||
|
||||
// tagPrefixOf is the prefix Go gives the tags of a Go package in its repository (A-490e): the package's directory in the
|
||||
// repository followed by "/v" ("go/v" for the sdk, whose Go module lives in go/), "v" at the root; none when the package's
|
||||
// name does not lie in the repository.
|
||||
func tagPrefixOf(repository string, a meshcatalog.Artifact) (string, bool) {
|
||||
if a.Package == nil || a.Package.Ecosystem != EcosystemGo || repository == "" {
|
||||
return "", false
|
||||
}
|
||||
i := strings.Index(a.Package.Name, "/"+repository)
|
||||
if i < 0 {
|
||||
return "", false
|
||||
}
|
||||
switch rest := a.Package.Name[i+1+len(repository):]; {
|
||||
case rest == "":
|
||||
return "v", true
|
||||
case strings.HasPrefix(rest, "/"):
|
||||
return rest[1:] + "/v", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// FloorModules are the modules of the newest trunk revision the versions hold none of and that name a package a floor can be read
|
||||
// of; each is filled by its own FillFloor, so one unanswered read holds no other module's floor (A-490i).
|
||||
func (ix *Indexer) FloorModules(ctx context.Context) ([]string, error) {
|
||||
if ix.Versions == nil || ix.Revisions == nil {
|
||||
return nil, errors.New("the state versions or the catalog graph is not kept here")
|
||||
}
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, errors.New("the indexer keeps no trunk revision yet: the floor waits for its first")
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !slices.ContainsFunc(e.Artifacts, func(a meshcatalog.Artifact) bool {
|
||||
return a.Package != nil && (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm)
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
if have, err := ModuleVersions(ctx, ix.Versions, e.Module); err != nil {
|
||||
return nil, err
|
||||
} else if len(have) == 0 {
|
||||
out = append(out, e.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FillFloor fills one module's floor, the highest version among its packages' tags and holders' versions (A-490g). Its reads are
|
||||
// kept calls made through mesh-sdk's form, which asks again through every transport fault; a read it gives up on (its own
|
||||
// pause spent) is asked again at once, never given up, so the read has no bound (A-490i). The first unanswered try starts the
|
||||
// module's clock: five minutes later the finding call-unanswered is written, and cleared when a read answers. A read the
|
||||
// callee answered with a failure ends the fill, left for the next start or revision. One fill of a module runs at a time.
|
||||
func (ix *Indexer) FillFloor(ctx context.Context, module string, src FloorSources) error {
|
||||
if _, running := ix.floors.LoadOrStore(module, true); running {
|
||||
return nil
|
||||
}
|
||||
defer ix.floors.Delete(module)
|
||||
// Whatever way the fill ends but by the end of ctx, the module's call is no longer unanswered: it answered, or it left the
|
||||
// trunk, or a release gave it a version, or the callee answered with a failure (ADR 0668 §3: the finding clears when the call
|
||||
// is answered), and FloorModules never picks the module again to clear it.
|
||||
defer func() {
|
||||
if ctx.Err() == nil {
|
||||
ix.floorAnswered(ctx, module)
|
||||
}
|
||||
}()
|
||||
for ctx.Err() == nil {
|
||||
entry, found, err := ix.newestEntry(ctx, module)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
if have, err := ModuleVersions(ctx, ix.Versions, module); err != nil || len(have) > 0 {
|
||||
return err
|
||||
}
|
||||
first := ix.now() // the clock of the module's unanswered call starts at its try, not when the form gives up
|
||||
top, err := ix.floorOf(ctx, entry, src)
|
||||
if err == nil {
|
||||
return ix.keepFloor(ctx, module, top)
|
||||
}
|
||||
if !busretry.Undelivered(err) && !busretry.TimedOut(err) {
|
||||
return err // answered with a failure: the fill ends, said by the caller
|
||||
}
|
||||
ix.floorUnanswered(ctx, module, first, err)
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (ix *Indexer) newestEntry(ctx context.Context, module string) (meshcatalog.CatalogEntry, bool, error) {
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil || !found {
|
||||
return meshcatalog.CatalogEntry{}, false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module == module {
|
||||
return e, true, nil
|
||||
}
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, false, nil
|
||||
}
|
||||
|
||||
// floorOf reads the highest version of the entry's packages; "" when none is held.
|
||||
func (ix *Indexer) floorOf(ctx context.Context, e meshcatalog.CatalogEntry, src FloorSources) (string, error) {
|
||||
top := ""
|
||||
take := func(vs []string, prefix string) {
|
||||
for _, t := range vs {
|
||||
if !strings.HasPrefix(t, prefix) {
|
||||
continue
|
||||
}
|
||||
if v := strings.TrimPrefix(t, prefix); versionOK(v) && (top == "" || compareVersions(v, top) > 0) {
|
||||
top = v
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, a := range e.Artifacts {
|
||||
if a.Package == nil {
|
||||
continue
|
||||
}
|
||||
if prefix, ok := tagPrefixOf(e.Repository, a); ok && src.Tags != nil {
|
||||
tags, err := src.Tags(ctx, e.Repository, prefix)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the tags of %s: %w", e.Repository, err)
|
||||
}
|
||||
take(tags, prefix)
|
||||
}
|
||||
if (a.Package.Ecosystem == EcosystemGo || a.Package.Ecosystem == EcosystemNpm) && src.Holder != nil {
|
||||
vs, err := src.Holder(ctx, a.Package.Ecosystem, a.Package.Name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the versions of %s %s: %w", a.Package.Ecosystem, a.Package.Name, err)
|
||||
}
|
||||
take(vs, "")
|
||||
}
|
||||
}
|
||||
return top, nil
|
||||
}
|
||||
|
||||
func (ix *Indexer) keepFloor(ctx context.Context, module, top string) error {
|
||||
if top == "" {
|
||||
return nil
|
||||
}
|
||||
now := ix.now().UTC()
|
||||
data, err := json.Marshal(meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: top, At: &now})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, top), data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return fmt.Errorf("the floor %s %s is not kept: %w", module, top, err)
|
||||
}
|
||||
ix.logf("versions: %s has none, filled with %s, the highest of its packages' tags and holders' versions", module, top)
|
||||
return nil
|
||||
}
|
||||
|
||||
func floorKey(module string) string { return FloorCall + "." + module }
|
||||
|
||||
// FloorSinceKey is the key of the state heard that holds when a module's floor call was first unanswered.
|
||||
func FloorSinceKey(module string) string { return floorSinceKey(module) }
|
||||
|
||||
func floorSinceKey(module string) string { return "call-since." + keys.EscapeName(floorKey(module)) }
|
||||
|
||||
// floorUnanswered writes call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state).
|
||||
func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first time.Time, cause error) {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, floorSinceKey(module), first)
|
||||
if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter {
|
||||
return
|
||||
}
|
||||
at := since.UTC().Format(time.RFC3339)
|
||||
f := ix.observed(mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered,
|
||||
fmt.Sprintf("the floor of %s's versions is unanswered since %s", module, at),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The floor of %s's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so it has no version.", module, at, cause),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("floor: the finding %s is not written: %v", floorKey(module), err)
|
||||
}
|
||||
}
|
||||
|
||||
func (ix *Indexer) floorAnswered(ctx context.Context, module string) {
|
||||
ix.answered(ctx, floorSinceKey(module))
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered, "the floor's reads answer"); err != nil {
|
||||
ix.logf("floor: the finding %s is not cleared: %v", floorKey(module), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func versionOK(v string) bool { _, ok := versionParts(v); return ok }
|
||||
@@ -120,9 +120,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
|
||||
}
|
||||
}
|
||||
if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
|
||||
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
} else if found && !isFloor(raw) {
|
||||
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact)
|
||||
return nil
|
||||
}
|
||||
@@ -212,7 +212,14 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
if err := ix.Versions.Create(ctx, key, data); errors.Is(err, ErrConflict) {
|
||||
// A floor entry (A-490h) holds no artifacts: this publish is that version's release, so it takes the entry's place.
|
||||
if raw, rev, found, gerr := ix.Versions.Get(ctx, key); gerr == nil && found && isFloor(raw) {
|
||||
if err := ix.Versions.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if late {
|
||||
@@ -339,3 +346,9 @@ func OutdatedTool(ix *Indexer) stdio.Tool {
|
||||
return ix.Outdated(context.Background(), in)
|
||||
}}
|
||||
}
|
||||
|
||||
// isFloor says whether a kept version is a floor entry (A-490h): it holds no artifact, as no release does.
|
||||
func isFloor(raw []byte) bool {
|
||||
var v meshcatalog.VersionEntry
|
||||
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
|
||||
}
|
||||
|
||||
@@ -534,3 +534,187 @@ func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
|
||||
t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §412 as A-490c, A-490d and A-490e amend it: a module the versions hold none of is filled from the highest tag of its
|
||||
// Go package in the repository it was taken in from, per module and not once; a forge that does not answer fills nothing and
|
||||
// raises call-unanswered after five minutes.
|
||||
func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
|
||||
w := newFindings()
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"mesh-sdk": {Module: "mesh-sdk", Repository: "novox/mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}},
|
||||
{Ref: "mesh-sdk:ts", Package: &meshcatalog.ArtifactPackage{Ecosystem: "npm", Name: "@novox/mesh-sdk"}}}},
|
||||
"root": {Module: "root", Repository: "novox/root", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "root:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/root"}}}},
|
||||
"held": {Module: "held", Repository: "novox/held", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "held:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/held"}}}},
|
||||
"image": {Module: "image", Repository: "novox/image", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "image:image", Kind: "image"}}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("held", "0.2.0"), []byte(`{"v":1,"module":"held","version":"0.2.0"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tries := 0
|
||||
var asked []string
|
||||
read := func(_ context.Context, repo, prefix string) ([]string, error) {
|
||||
asked = append(asked, repo+" "+prefix)
|
||||
if repo == "novox/mesh-sdk" {
|
||||
// the forge does not answer the sdk's tags for six minutes, a minute at a time: the fill asks again, never gives up
|
||||
tries++
|
||||
now = now.Add(time.Minute)
|
||||
if tries == 5 && len(w.Held()) != 0 {
|
||||
t.Error("the finding is written before five minutes from the first unanswered try")
|
||||
}
|
||||
if tries <= 6 {
|
||||
return nil, errors.New("no responders available for request")
|
||||
}
|
||||
if len(w.Held()) != 1 || w.Held()[0].ID != FloorCall+".mesh-sdk" {
|
||||
t.Errorf("the findings after six minutes: %+v", w.Held())
|
||||
}
|
||||
return []string{"go/v0.1.9", "go/v0.1.16", "go/v0.1.12", "go/vnext"}, nil
|
||||
}
|
||||
if repo == "novox/root" {
|
||||
return []string{"v1.2.3", "vendor-x"}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
holder := func(_ context.Context, eco, name string) ([]string, error) {
|
||||
if eco == "npm" && name == "@novox/mesh-sdk" {
|
||||
return []string{"0.1.18", "0.1.17"}, nil // A-490g: the npm holder is above the Go tags
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
src := FloorSources{Tags: read, Holder: holder}
|
||||
modules, err := ix.FloorModules(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(modules, []string{"mesh-sdk", "root"}) {
|
||||
t.Fatalf("the modules with a floor to read: %v (held has a version, image no package)", modules)
|
||||
}
|
||||
// each module's floor is read on its own: root is filled whatever the sdk's read waits for
|
||||
if err := ix.FillFloor(ctx, "root", src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.FillFloor(ctx, "mesh-sdk", src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Errorf("the finding stands after the forge answered: %+v", w.Held())
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk"); len(v) != 1 || v[0].Version != "0.1.18" {
|
||||
t.Errorf("mesh-sdk's floor: %+v (the highest of its Go tags 0.1.16 and the npm holder's 0.1.18)", v)
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "root"); len(v) != 1 || v[0].Version != "1.2.3" {
|
||||
t.Errorf("a root module's floor: %+v", v)
|
||||
}
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "held"); len(v) != 1 || v[0].Version != "0.2.0" {
|
||||
t.Errorf("a module with a version is filled: %+v", v)
|
||||
}
|
||||
for _, a := range asked {
|
||||
if strings.HasPrefix(a, "novox/held") || strings.HasPrefix(a, "novox/image") {
|
||||
t.Errorf("tags read for a module with a version or no Go package: %s", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-490h: a floor entry holds no artifacts, so the package-published heard later at its version is that version's
|
||||
// release (the entry takes its artifacts and version-released is said), never dropped as released already.
|
||||
func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
|
||||
var said []meshcatalog.VersionReleasedEvent
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: newFindings(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now },
|
||||
SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, _ string) error {
|
||||
said = append(said, e)
|
||||
return nil
|
||||
}}
|
||||
entries := map[string]meshcatalog.CatalogEntry{"mod": {Module: "mod", Repository: "novox/mod", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "mod:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mod"}}}}}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := FloorSources{Tags: func(context.Context, string, string) ([]string, error) { return []string{"v0.3.0"}, nil }}
|
||||
if err := ix.FillFloor(ctx, "mod", src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "mod", Artifact: "mod:go", Version: "0.3.0", Digest: "sha256:aa", Branch: "main", Source: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, _ := ModuleVersions(ctx, ix.Versions, "mod")
|
||||
if len(v) != 1 || len(v[0].Artifacts) != 1 || v[0].Artifacts[0].Digest != "sha256:aa" || len(said) != 1 {
|
||||
t.Fatalf("the floor 0.3.0 and its publish: versions %+v, version-released said %d", v, len(said))
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0668 §3, design 47 G A-490i: the finding call-unanswered of a module's floor clears, with its call-since key, on every way the
|
||||
// fill ends but the end of its context: a release gave the module a version, the module left the trunk, or the callee answered with a
|
||||
// failure (FloorModules never picks the module again to clear it).
|
||||
func TestTheFloorFindingClearsOnEveryExitOfTheFill(t *testing.T) {
|
||||
for name, exit := range map[string]func(ctx context.Context, t *testing.T, ix *Indexer) error{
|
||||
"a release gave it a version": func(ctx context.Context, t *testing.T, ix *Indexer) error {
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey("mod", "0.5.0"), []byte(`{"v":1,"module":"mod","version":"0.5.0","artifacts":[{"artifact":"mod:go"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return errors.New("no responders available for request")
|
||||
},
|
||||
"it left the trunk": func(ctx context.Context, t *testing.T, ix *Indexer) error {
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 2, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return errors.New("no responders available for request")
|
||||
},
|
||||
"the callee answered with a failure": func(context.Context, *testing.T, *Indexer) error { return errors.New("the forge refused") },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
|
||||
w := newFindings()
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }}
|
||||
entries := map[string]meshcatalog.CatalogEntry{"mod": {Module: "mod", Repository: "novox/mod", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "mod:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mod"}}}}}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tries := 0
|
||||
read := func(context.Context, string, string) ([]string, error) {
|
||||
tries++
|
||||
now = now.Add(time.Minute)
|
||||
if tries <= 6 {
|
||||
return nil, errors.New("no responders available for request")
|
||||
}
|
||||
if len(w.Held()) != 1 {
|
||||
t.Fatalf("the finding is not written before the exit: %+v", w.Held())
|
||||
}
|
||||
return nil, exit(ctx, t, ix)
|
||||
}
|
||||
_ = ix.FillFloor(ctx, "mod", FloorSources{Tags: read})
|
||||
if h := w.Held(); len(h) != 0 {
|
||||
t.Errorf("the finding stands after the fill ended: %+v", h)
|
||||
}
|
||||
if _, _, found, _ := ix.Heard.Get(ctx, FloorSinceKey("mod")); found {
|
||||
t.Error("the call-since key stands after the fill ended")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
@@ -416,7 +416,7 @@ func TestEveryGiteaToolTheCodeCallsIsInInvokes(t *testing.T) {
|
||||
if err != nil || len(files) == 0 {
|
||||
t.Fatalf("no program files: %v", err)
|
||||
}
|
||||
verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch}
|
||||
verbs := map[string]string{"VerbTrees": git.VerbTrees, "VerbFile": git.VerbFile, "VerbContains": git.VerbContains, "VerbOpenPulls": git.VerbOpenPulls, "VerbBranch": git.VerbBranch, "VerbTags": git.VerbTags}
|
||||
forge, asked := regexp.MustCompile(`gitea\.gitea_[a-z_]+`), regexp.MustCompile(`git\.Verb[A-Za-z]+`)
|
||||
seen := 0
|
||||
for _, f := range files {
|
||||
|
||||
@@ -167,6 +167,9 @@
|
||||
],
|
||||
"invokes": [
|
||||
"seat:git.trees",
|
||||
"seat:git.tags",
|
||||
"seat:go-package-registry.versions",
|
||||
"seat:npm-package-registry.versions",
|
||||
"seat:git.file",
|
||||
"seat:git.contains",
|
||||
"seat:git.open-pulls",
|
||||
|
||||
Reference in New Issue
Block a user