Merge branch 'feat/47-G-indexer6b' into feat/47-G-join8
# Conflicts: # modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go # modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go # modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go # modules/mesh-delivery-indexer/internal/index/pending.go # modules/mesh-delivery-indexer/internal/index/pending_test.go # modules/mesh-delivery-indexer/manifest_test.go
This commit is contained in:
@@ -288,7 +288,9 @@ func main() {
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}))
|
||||
tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
|
||||
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
|
||||
index.InputTool(keptRead{stdio.State(index.InputsState)}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}))
|
||||
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"})
|
||||
return err
|
||||
|
||||
@@ -1474,3 +1474,74 @@ func TestIndexerKeepsWhatAWantAndARunningBuildNameAndLetsAGoneOneLeave(t *testin
|
||||
t.Errorf("named holds %v after the want of n2 was said removed, want n1.avahi alone", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision: the seat's verb source lists a second watched branch,
|
||||
// beta, with the merge of mesh-catalog it names as its head (design 47 G §13, §14, §291, A-93); the indexer started with the
|
||||
// branch listed reads source.beta, takes the head's merged message in as a pure computation, and keeps branch.beta.1, which
|
||||
// the branch follower (index.BranchView, as the branch's builds read it) reads; the trunk's modules are left.
|
||||
func TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision(t *testing.T) {
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "beta", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "betatree"}, Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
|
||||
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
|
||||
}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
seedBuildModules(t, ctx, js, nil)
|
||||
source := func(m *nats.Msg) {
|
||||
var ask struct{ Branch string }
|
||||
_ = json.Unmarshal(m.Data, &ask)
|
||||
answer := meshdelivery.SourceAnswer{Branch: ask.Branch}
|
||||
switch ask.Branch {
|
||||
case "":
|
||||
answer.Branches = []string{"main", "beta"}
|
||||
case "beta":
|
||||
answer.Group, answer.Revision = "g", 9
|
||||
answer.Repositories = []meshdelivery.SourceHead{{Repository: "novox/mesh-catalog", Head: merge, Trunk: "beta"}}
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"result": answer, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
bridge := func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data)}, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
for subject, h := range map[string]nats.MsgHandler{
|
||||
"mesh.seat.mesh-delivery.call.source.>": source, "mesh.seat.mesh-delivery.tool.source": source,
|
||||
"mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>": bridge, "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message": bridge,
|
||||
} {
|
||||
if _, err := js.Conn().Subscribe(subject, h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects)
|
||||
var view index.Store
|
||||
var at int64
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && at == 0; time.Sleep(50 * time.Millisecond) {
|
||||
if v, _, n, err := index.BranchView(ctx, objects, "beta"); err == nil {
|
||||
view, at = v, n
|
||||
}
|
||||
}
|
||||
if at != 1 {
|
||||
t.Fatalf("the branch follower reads beta at revision %d within 20 s of the start, want 1", at)
|
||||
}
|
||||
value, _, found, err := view.Get(ctx, "avahi")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("branch.beta.1 holds no avahi (%v)", err)
|
||||
}
|
||||
if m, err := meshcatalogclient.DecodeModule(value); err != nil || m.Commit != merge || m.Inputs2 == nil {
|
||||
t.Errorf("avahi at the branch's head = %+v (%v), want commit %s with its inputs", m, err, merge)
|
||||
}
|
||||
if e, err := modules.Get(ctx, "avahi"); err == nil {
|
||||
t.Errorf("modules.avahi = %s, want the trunk's modules left", e.Value())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,10 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -397,6 +400,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// appears; until then indexer_requirements answers none, and the builder's floor waits on its answer.
|
||||
requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf)
|
||||
// The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes.
|
||||
// The state heard (A-80): the forge messages handled (key forge) and what the indexer heard and compares later (the time a call is
|
||||
// unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version).
|
||||
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "what the indexer heard is not kept", logger.Printf)}
|
||||
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
|
||||
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
|
||||
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
|
||||
@@ -430,6 +436,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
})
|
||||
readWhenBound(seatReader, seatDefs, marks)
|
||||
var fillFollow atomic.Pointer[versionsFill]
|
||||
// Kept while named (design 47 G §675): the want and the running build that name a fingerprint, heard below.
|
||||
named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.NamedBucket, "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf)
|
||||
ix := &index.Indexer{NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
@@ -437,12 +444,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
// Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go).
|
||||
SayCatalogRevision: refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow),
|
||||
SayCatalogRevision: afterRevision(refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), &fillFollow),
|
||||
SayFill: sayFill(conn, cred),
|
||||
// The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released.
|
||||
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)},
|
||||
BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.BuildEdgesBucket, "no build of another watched branch keeps its edges", logger.Printf)},
|
||||
Inputs: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)},
|
||||
SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
@@ -453,10 +465,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.RegistrationsBucket, "no registration is kept or written", logger.Printf)},
|
||||
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)},
|
||||
routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf)
|
||||
registry.Pruned = ix.PruneBranchBuilds
|
||||
ix.Registry = registry
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
ix.SeedModules = func(context.Context) ([]index.SeedModule, error) {
|
||||
return nil, errors.New("the first registrations are not composed by this build: what the build seats answer steps with is not read yet (design question)")
|
||||
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
|
||||
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKept), logf: logger.Printf}
|
||||
fillFollow.Store(fill)
|
||||
fill.kick()
|
||||
ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) {
|
||||
r := seatReader.Reader()
|
||||
if r == nil {
|
||||
return nil, errors.New("the seat list is not read yet: the seed waits for it")
|
||||
}
|
||||
return ix.SeedModulesFrom(ctx, r.Definitions(), stepsVia(calls.CallKept))
|
||||
}
|
||||
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
|
||||
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
|
||||
@@ -493,7 +514,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
|
||||
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
|
||||
// meanwhile is read after.
|
||||
source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, logf: logger.Printf}
|
||||
source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf}
|
||||
go source.catchUp(ctx)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// assignments is settled first, so the toolchains settle through the gate already knowing it.
|
||||
@@ -666,6 +687,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
_ = m.Ack() // nothing else the indexer consumes rides the shared stream
|
||||
}
|
||||
}
|
||||
if err := hub.Handle(meshbuilds.SubjectPackagePublished, func(m jetstream.Msg) { heardPackagePublished(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(meshbuilds.SubjectToolchainAgreed, func(m jetstream.Msg) { heardToolchainAgreed(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(meshwants.SubjectVersionDelivered, func(m jetstream.Msg) { heardVersionDelivered(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(meshwants.SubjectWaitEnded, func(m jetstream.Msg) { heardWaitEnded(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -726,7 +762,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
|
||||
// What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds
|
||||
// (issue 1790): until then no message is written there, and the rest of the indexer runs.
|
||||
heard := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
|
||||
heard := heardStore
|
||||
// Each change of the states its readers read is said on its views family (ADR 0615 §5), from the mark each state was
|
||||
// said up to; a first view (the tools bundle) answers at that mark.
|
||||
startViews(ctx, js, cred, marks, writer,
|
||||
|
||||
@@ -191,6 +191,9 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(
|
||||
s = &sourceFollow{
|
||||
read: func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
order = append(order, "read "+branch)
|
||||
if branch == "" { // given no branch the seat lists the branches it keeps a source of (A-93)
|
||||
return meshdelivery.SourceAnswer{Branches: []string{"main", "beta", "feat=2e1"}}, nil
|
||||
}
|
||||
if revision == 0 {
|
||||
return meshdelivery.SourceAnswer{}, fmt.Errorf("source.main is not readable")
|
||||
}
|
||||
@@ -205,15 +208,21 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(
|
||||
order = append(order, fmt.Sprintf("apply %s %d", a.Branch, a.Revision))
|
||||
return nil
|
||||
},
|
||||
applyBranch: func(_ context.Context, branch string, a meshdelivery.SourceAnswer) error {
|
||||
order = append(order, fmt.Sprintf("branch %s %d", branch, a.Revision))
|
||||
return nil
|
||||
},
|
||||
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) },
|
||||
}
|
||||
s.catchUp(context.Background())
|
||||
want := []string{"read main", "apply main 4", "read main", "apply main 4"} // once, after the catch-up; beta is left
|
||||
// The trunk, then each other watched branch the seat lists (a name unescaped), then what was heard meanwhile once each,
|
||||
// beta among them: main through the take-in, every other branch into its own revision (A-93).
|
||||
want := []string{"read main", "apply main 4", "read ", "read beta", "branch beta 4", "read feat=2e1", "branch feat.1 4", "read main", "apply main 4", "read beta", "branch beta 4"}
|
||||
if !reflect.DeepEqual(order, want) {
|
||||
t.Errorf("order = %v, want %v", order, want)
|
||||
}
|
||||
s.moved(context.Background(), "main") // after the catch-up: read at once
|
||||
if len(order) != 6 {
|
||||
if len(order) != 13 {
|
||||
t.Errorf("a source-moved after the catch-up was not read at once: %v", order)
|
||||
}
|
||||
order, revision = nil, 0
|
||||
|
||||
@@ -237,7 +237,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
|
||||
}
|
||||
|
||||
// trunkBranch is the branch pull requests are open into.
|
||||
const trunkBranch = "main"
|
||||
const trunkBranch = index.TrunkBranch
|
||||
|
||||
// openVia asks the forge which pull requests into the trunk are open now (the seat git's open-pulls), every repository's.
|
||||
// An answer that lacks the list is refused, never read as none.
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
@@ -40,17 +41,31 @@ func sourceVia(call toolCaller) func(branch string) (meshdelivery.SourceAnswer,
|
||||
type sourceFollow struct {
|
||||
read func(branch string) (meshdelivery.SourceAnswer, error)
|
||||
apply func(context.Context, meshdelivery.SourceAnswer) error
|
||||
logf func(string, ...any)
|
||||
// applyBranch takes a watched branch other than the trunk's in (design 47 G §291, A-93); branch is its name, unescaped.
|
||||
applyBranch func(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
caught bool
|
||||
waiting map[string]bool
|
||||
}
|
||||
|
||||
// catchUp reads the trunk branch's source revision and applies it, then the branches a source-moved named meanwhile. A
|
||||
// read that fails is said; the catch-up ends all the same, and the next source-moved reads again.
|
||||
// catchUp reads the trunk branch's source revision and applies it, then every other branch the seat keeps a source of (the
|
||||
// answer given no branch lists them, A-93), then the branches a source-moved named meanwhile. A read that fails is said;
|
||||
// the catch-up ends all the same, and the next source-moved reads again.
|
||||
func (s *sourceFollow) catchUp(ctx context.Context) {
|
||||
s.take(ctx, trunkBranch)
|
||||
var others []string
|
||||
if list, err := s.read(""); err != nil {
|
||||
s.logf("the branches the source names are not read, and each is read at its next source-moved: %v", err)
|
||||
} else {
|
||||
others = list.Branches
|
||||
}
|
||||
for _, branch := range others {
|
||||
if branch != trunkBranch {
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.caught = true
|
||||
branches := s.waiting
|
||||
@@ -76,17 +91,28 @@ func (s *sourceFollow) moved(ctx context.Context, branch string) {
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
|
||||
// take reads one branch's source revision and applies it. A branch other than the trunk's is left until the indexer keeps
|
||||
// a catalog graph per watched branch.
|
||||
// take reads one branch's source revision (branch as the key name source.<branch> has it) and applies it: the trunk's
|
||||
// through the take-in, another watched branch's into its own revision (A-93).
|
||||
func (s *sourceFollow) take(ctx context.Context, branch string) {
|
||||
if branch != trunkBranch {
|
||||
return
|
||||
}
|
||||
a, err := s.read(branch)
|
||||
if err != nil {
|
||||
s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if branch != trunkBranch {
|
||||
name, err := keys.UnescapeName(branch)
|
||||
if err != nil {
|
||||
s.logf("source.%s names no branch: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if s.applyBranch == nil {
|
||||
return
|
||||
}
|
||||
if err := s.applyBranch(ctx, name, a); err != nil {
|
||||
s.logf("source.%s at revision %d could not be taken into the branch's revision: %v", branch, a.Revision, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := s.apply(ctx, a); err != nil {
|
||||
s.logf("source.%s at revision %d could not be taken into the modules: %v", branch, a.Revision, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// stepsVia asks a build seat's verb steps through call (process.Conn.CallKept, ADR 0668 §1) and reads its answer (design
|
||||
// 47 G §704); the seats' verbs have one form, so nodebuildgo's contract reads each seat's answer.
|
||||
func stepsVia(call toolCaller) index.StepsCaller {
|
||||
return func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) {
|
||||
verb := "seat:" + seat + "." + index.StepsVerb
|
||||
raw, err := call(verb, in)
|
||||
if err != nil {
|
||||
return nodebuildgo.StepsAnswer{}, err
|
||||
}
|
||||
var a nodebuildgo.StepsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the answer of %s is not its steps: %w", verb, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// saySeatEvent publishes body as the seat mesh-catalog's event under the bus's duplicate id, as every event of the seat is
|
||||
// said: published, never asked, and a refusal said on the connection's error handler.
|
||||
func saySeatEvent(conn *nats.Conn, cred process.Account, event, id string, body any) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
return conn.PublishMsg(msg)
|
||||
}
|
||||
|
||||
// sayVersionReleased says version-released (design 47 G §349) on mesh.seat.mesh-catalog.event.version-released.
|
||||
func sayVersionReleased(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionReleasedEvent, string) error {
|
||||
return func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error {
|
||||
return saySeatEvent(conn, cred, meshcatalog.EventVersionReleased, id, e)
|
||||
}
|
||||
}
|
||||
|
||||
// heardPackagePublished takes one package-published of the seat mesh-builds into the catalog's versions (design 47 G §349):
|
||||
// one that does not decode is said and acknowledged; one that cannot be kept is left, so the bus gives it again.
|
||||
func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var p meshbuilds.PackagePublished
|
||||
if err := json.Unmarshal(m.Data(), &p); err != nil {
|
||||
logf("%s holds what is no package-published, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if err := ix.HearPackagePublished(ctx, p); err != nil {
|
||||
logf("versions: the package-published of %s %s could not be kept, so the bus gives it again: %v", p.Artifact, p.Version, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// heardToolchainAgreed keeps the newest toolchain-agreed of its language (A-68); one that does not decode is said and
|
||||
// acknowledged, one that cannot be kept is left, so the bus gives it again.
|
||||
func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var t meshbuilds.ToolchainAgreed
|
||||
if err := json.Unmarshal(m.Data(), &t); err != nil {
|
||||
logf("%s holds what is no toolchain-agreed, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, t); err != nil {
|
||||
logf("versions: the toolchain-agreed of %s could not be kept, so the bus gives it again: %v", t.Language, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// heardVersionDelivered writes the delivered release's entries of kinds 2, 4 and 6 and keeps the newest delivered version of
|
||||
// its module (design 47 G §374, A-68), as heardToolchainAgreed keeps its event.
|
||||
func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var d meshwants.VersionDeliveredEvent
|
||||
if err := json.Unmarshal(m.Data(), &d); err != nil {
|
||||
logf("%s holds what is no version-delivered, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if err := ix.HearVersionDelivered(ctx, d); err != nil {
|
||||
logf("versions: the version-delivered of %s %s could not be kept, so the bus gives it again: %v", d.Module, d.Version, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// heardWaitEnded keeps the act of an acting part's ended wait for kind 5 (A-78 a), as heardToolchainAgreed keeps its event.
|
||||
func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var w meshwants.WaitEndedEvent
|
||||
if err := json.Unmarshal(m.Data(), &w); err != nil {
|
||||
logf("%s holds what is no wait-ended, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if err := ix.HearWaitEnded(ctx, w); err != nil {
|
||||
logf("versions: the wait-ended of %s for %s could not be kept, so the bus gives it again: %v", w.Part, w.Module, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// heardStepEnded deletes the input kept for the run ask a build seat's step-ended names (design 47 G §354, A-78 d).
|
||||
func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var e nodebuildgo.StepEnded
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if err := ix.HearInputEnded(ctx, e); err != nil {
|
||||
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// sayVersionsFilled says versions-filled (design 47 G §372) once, on the seat's stream that keeps it with no age limit.
|
||||
func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionsFilledEvent, string) error {
|
||||
return func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error {
|
||||
return saySeatEvent(conn, cred, meshcatalog.EventVersionsFilled, id, e)
|
||||
}
|
||||
}
|
||||
|
||||
// versionsVia asks the builder's tool builder_versions for one name as a kept call (ADR 0668 §1, §2) and reads its answer.
|
||||
func versionsVia(call toolCaller) index.VersionsAsker {
|
||||
return func(_ context.Context, name string) (index.BuilderVersionsAnswer, error) {
|
||||
var a index.BuilderVersionsAnswer
|
||||
raw, err := call(index.BuilderVersionsCall, map[string]any{"name": name})
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the answer of %s is not a name's versions: %w", index.BuilderVersionsCall, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each
|
||||
// catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next.
|
||||
type versionsFill struct {
|
||||
ix *index.Indexer
|
||||
ctx context.Context
|
||||
ask index.VersionsAsker
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
func (f *versionsFill) kick() {
|
||||
go func() {
|
||||
if err := f.ix.FillVersions(f.ctx, f.ask); err != nil {
|
||||
f.logf("versions: the first fill waits: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// afterRevision runs the fill's kick after each catalog revision kept, once the fill is set.
|
||||
func afterRevision(say func(context.Context, meshcatalog.CatalogRevisionEvent) error, fill *atomic.Pointer[versionsFill]) func(context.Context, meshcatalog.CatalogRevisionEvent) error {
|
||||
return func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
err := say(ctx, e)
|
||||
if f := fill.Load(); f != nil {
|
||||
f.kick()
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// placementVia asks the seat mesh-module-assigner's verb placement for one module (A-62), a read.
|
||||
func placementVia(call toolCaller) index.PlacementAsker {
|
||||
return func(_ context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) {
|
||||
raw, err := call(meshmoduleassigner.Seat+"."+meshmoduleassigner.VerbPlacement, map[string]any{"module": module})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s.%s: %w", meshmoduleassigner.Seat, meshmoduleassigner.VerbPlacement, err)
|
||||
}
|
||||
var a meshmoduleassigner.PlacementAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of placement is not a module's placements: %w", err)
|
||||
}
|
||||
return a.Placements, nil
|
||||
}
|
||||
}
|
||||
|
||||
// runningWholeVia reads every subject's last message of the node-engine seat's stream running through an ordered consumer made for
|
||||
// that one read (design 47 G §556), by node and module.
|
||||
func runningWholeVia(js jetstream.JetStream) index.RunningReader {
|
||||
return func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) {
|
||||
stream, err := js.Stream(ctx, nodeengine.StreamRunning)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the stream %s cannot be read whole: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the ordered consumer for the whole read of %s cannot be made: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
out := map[string]map[string]nodeengine.Running{}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(256)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
var r nodeengine.Running
|
||||
if json.Unmarshal(m.Data(), &r) != nil || r.Node == "" || r.Module == "" {
|
||||
continue
|
||||
}
|
||||
if out[r.Node] == nil {
|
||||
out[r.Node] = map[string]nodeengine.Running{}
|
||||
}
|
||||
out[r.Node][r.Module] = r
|
||||
}
|
||||
if err := batch.Error(); err != nil {
|
||||
return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
if n == 0 {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,7 +70,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The builds of a watched branch other than the first release no version (design 47 G §371); the indexer keeps each one's
|
||||
// edge entries in the state build-edges, under <module>.<branch>, and what the branch's newest build of the module
|
||||
// published as heard, branch-build.<module>.<branch> in the state heard (§374, A-83).
|
||||
const (
|
||||
BuildEdgesState = "build-edges"
|
||||
BuildEdgesBucket = Module + "_" + BuildEdgesState
|
||||
// BranchBuildPrefix opens branch-build.<module>.<branch>, each part escaped as a name (A-80).
|
||||
BranchBuildPrefix = "branch-build."
|
||||
)
|
||||
|
||||
// BranchBuild is what a branch-build key holds: the newest build of a module on a watched branch that was heard published,
|
||||
// its source revision, the fingerprint of the branch's catalog entry at that revision and each artifact's digest.
|
||||
type BranchBuild struct {
|
||||
V int64 `json:"v"`
|
||||
Version string `json:"version"`
|
||||
Source int64 `json:"source"`
|
||||
Fp string `json:"fp"`
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// BranchBuildKey is the key of the state heard of a module's newest build on a branch.
|
||||
func BranchBuildKey(module, branch string) string {
|
||||
return BranchBuildPrefix + keys.EscapeName(module) + "." + keys.EscapeName(branch)
|
||||
}
|
||||
|
||||
// BuildEdgesKey is the key of the state build-edges of a module's build on a branch, each part escaped as a name.
|
||||
func BuildEdgesKey(module, branch string) string {
|
||||
return meshcatalog.BuildEdgesByModuleBranchKey(keys.EscapeName(module), keys.EscapeName(branch))
|
||||
}
|
||||
|
||||
// branchEntryAt is the module's entry in the newest revision of the branch taken in from that source revision (A-83: the
|
||||
// revision whose source the build names, which the indexer holds itself).
|
||||
func (ix *Indexer) branchEntryAt(ctx context.Context, module, branch string, source int64) (meshcatalog.CatalogEntry, error) {
|
||||
if ix.Revisions == nil {
|
||||
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry says the build's fingerprint")
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogEntry{}, err
|
||||
}
|
||||
number := TrunkRevisionOf
|
||||
if !isTrunkBranch(branch) {
|
||||
number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) }
|
||||
}
|
||||
var numbers []int64
|
||||
for _, n := range names {
|
||||
if r, ok := number(n); ok {
|
||||
numbers = append(numbers, r)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) })
|
||||
for _, n := range numbers {
|
||||
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: n})
|
||||
rev, entries, err := ReadRevision(ctx, ix.Revisions, name)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogEntry{}, err
|
||||
}
|
||||
if rev.Source != source {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module == module {
|
||||
return e, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, fmt.Errorf("the indexer keeps no revision of the branch %s at source revision %d with an entry of %s", branch, source, module)
|
||||
}
|
||||
|
||||
// hearBranchBuild takes the package-published of a build of a watched branch other than the first (A-83): the build is kept as
|
||||
// heard, and its edge entries are kept in build-edges, keyed by the fingerprint of the entry in the revision of that branch the
|
||||
// build's source names. A build of an older source revision than the one kept leaves what is kept.
|
||||
func (ix *Indexer) hearBranchBuild(ctx context.Context, p meshbuilds.PackagePublished) error {
|
||||
if ix.BuildEdges == nil {
|
||||
return errors.New("the state build-edges is not kept here")
|
||||
}
|
||||
entry, err := ix.branchEntryAt(ctx, p.Module, p.Branch, p.Source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.Fp == "" {
|
||||
ix.logf("build-edges: the entry of %s on %s at source revision %d has no fingerprint, so its build keeps none", p.Module, p.Branch, p.Source)
|
||||
return nil
|
||||
}
|
||||
at := ix.now().UTC()
|
||||
err = write(ctx, ix.Heard, BranchBuildKey(p.Module, p.Branch), func(raw []byte, found bool) ([]byte, bool, error) {
|
||||
var b BranchBuild
|
||||
if found && json.Unmarshal(raw, &b) == nil && b.Source > p.Source {
|
||||
return nil, false, nil
|
||||
}
|
||||
if !found || b.Source != p.Source || b.Fp != entry.Fp {
|
||||
b = BranchBuild{Artifacts: map[string]string{}}
|
||||
}
|
||||
b.V, b.Version, b.Source, b.Fp, b.At = 1, p.Version, p.Source, entry.Fp, at
|
||||
b.Artifacts[p.Artifact] = p.Digest
|
||||
data, err := json.Marshal(b)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
edges := releaseEdges(entry, p.Dependencies)
|
||||
return write(ctx, ix.BuildEdges, BuildEdgesKey(p.Module, p.Branch), func(raw []byte, found bool) ([]byte, bool, error) {
|
||||
be := meshcatalog.BuildEdges{V: 1, Fp: entry.Fp, Source: p.Source}
|
||||
if found {
|
||||
kept, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if kept.Source > p.Source {
|
||||
return nil, false, nil
|
||||
}
|
||||
if kept.Source == p.Source && kept.Fp == entry.Fp {
|
||||
be.Edges = kept.Edges
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
if !slices.ContainsFunc(be.Edges, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word }) {
|
||||
be.Edges = append(be.Edges, e)
|
||||
}
|
||||
}
|
||||
data, err := json.Marshal(be)
|
||||
return data, false, err
|
||||
})
|
||||
}
|
||||
|
||||
// PruneBranchBuilds forgets a module's builds of the branches its registration no longer watches beside its first
|
||||
// (registration-changed, §374): its build-edges keys and its branch-build keys of every other branch.
|
||||
func (ix *Indexer) PruneBranchBuilds(ctx context.Context, module string, watched []string) error {
|
||||
keep := map[string]bool{}
|
||||
if len(watched) > 1 {
|
||||
for _, b := range watched[1:] {
|
||||
keep[keys.EscapeName(b)] = true
|
||||
}
|
||||
}
|
||||
var errs []error
|
||||
forget := func(st Store, key string) {
|
||||
if _, rev, found, err := st.Get(ctx, key); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else if found {
|
||||
if err := st.Delete(ctx, key, rev); err != nil && !errors.Is(err, ErrConflict) {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ix.BuildEdges != nil {
|
||||
all, err := ix.BuildEdges.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range all {
|
||||
if m, b, ok := meshcatalog.ParseBuildEdgesByModuleBranchKey(k); ok && m == keys.EscapeName(module) && !keep[b] {
|
||||
forget(ix.BuildEdges, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ix.Heard != nil {
|
||||
all, err := ix.Heard.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prefix := BranchBuildPrefix + keys.EscapeName(module) + "."
|
||||
for _, k := range all {
|
||||
if b, ok := strings.CutPrefix(k, prefix); ok && !keep[b] {
|
||||
forget(ix.Heard, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// outdatedBranchBuilds are the behind entries of kind 1 of the builds of other watched branches (A-83): the digest a build
|
||||
// recorded for a package against the digest in the newest build of that package's module on the same branch, as heard, never a
|
||||
// released digest.
|
||||
func (ix *Indexer) outdatedBranchBuilds(ctx context.Context, in meshcatalog.OutdatedInput) ([]meshcatalog.Outdated, error) {
|
||||
if ix.BuildEdges == nil || ix.Heard == nil {
|
||||
return nil, nil
|
||||
}
|
||||
all, err := ix.BuildEdges.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(all)
|
||||
var out []meshcatalog.Outdated
|
||||
for _, k := range all {
|
||||
em, eb, ok := meshcatalog.ParseBuildEdgesByModuleBranchKey(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
module, err1 := keys.UnescapeName(em)
|
||||
branch, err2 := keys.UnescapeName(eb)
|
||||
if err1 != nil || err2 != nil || (in.Branch != "" && in.Branch != eb) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := ix.BuildEdges.Get(ctx, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
be, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the key %s of the state %s: %w", k, BuildEdgesState, err)
|
||||
}
|
||||
for _, e := range be.Edges {
|
||||
if e.Kind != 1 || (in.Dependency != "" && in.Dependency != e.Part) {
|
||||
continue
|
||||
}
|
||||
var dep BranchBuild
|
||||
raw, _, found, err := ix.Heard.Get(ctx, BranchBuildKey(strings.SplitN(e.Part, ":", 2)[0], branch))
|
||||
if err != nil || !found || json.Unmarshal(raw, &dep) != nil {
|
||||
continue
|
||||
}
|
||||
digest := dep.Artifacts[e.Part]
|
||||
if digest == "" || digest == e.Digest {
|
||||
continue
|
||||
}
|
||||
o := meshcatalog.Outdated{Module: module, Build: be.Fp, Kind: 1, Recorded: e.Version + " " + e.Digest, Newest: dep.Version + " " + digest,
|
||||
Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: branch}}
|
||||
if e.Stale {
|
||||
o.Act = "rebuild"
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// Design 47 G §374, A-83, A-85: a build of a watched branch other than the first releases no version; its edge entries are kept
|
||||
// in build-edges under <module>.<branch>, the fingerprint being the entry's in the branch's revision whose source the build
|
||||
// names; outdated compares kind 1 with the same branch's newest build, as heard; registration-changed prunes the branch's
|
||||
// keys when it is no longer watched.
|
||||
func TestABuildOfAnotherWatchedBranchKeepsItsEdgesAndIsOutdatedAgainstThatBranch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Versions: newMem(), Heard: newMem(), BuildEdges: newMem(), Revisions: newMemObjects(1 << 20), Registry: &Registry{Registrations: MemoryStore()}}
|
||||
for _, m := range []string{"lib", "app"} {
|
||||
raw, _ := json.Marshal(meshcatalog.Registration{Module: m, Branches: []string{"main", "beta"}})
|
||||
if err := ix.Registry.Registrations.Create(ctx, RegistrationKey(m), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) }
|
||||
keep := func(source int64, libFp, appFp string) {
|
||||
t.Helper()
|
||||
err := ix.KeepBranchEntries(ctx, "beta", source, map[string]meshcatalog.CatalogEntry{
|
||||
"lib": {Module: "lib", Fp: libFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}},
|
||||
"app": {Module: "app", Fp: appFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}},
|
||||
Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
keep(7, fp("a"), fp("b"))
|
||||
keep(8, fp("c"), fp("b"))
|
||||
pub := func(module, artifact, digest string, source int64, deps ...meshbuilds.Dependency) {
|
||||
t.Helper()
|
||||
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: module, Artifact: artifact, Digest: digest, Version: "0.1.1", Source: source, Branch: "beta", Dependencies: deps}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pub("lib", "lib:pkg", "d1", 7)
|
||||
pub("app", "app:program", "a1", 7, meshbuilds.Dependency{Kind: 1, Part: "lib:pkg", Version: "0.1.1", Digest: "d1"})
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "app"); len(v) != 0 {
|
||||
t.Errorf("a build of beta released %v", v)
|
||||
}
|
||||
raw, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta"))
|
||||
be, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw)
|
||||
if !found || err != nil || be.Fp != fp("b") || be.Source != 7 || len(be.Edges) != 1 || be.Edges[0].Digest != "d1" {
|
||||
t.Fatalf("build-edges of app on beta: %+v (found %v, %v), want the fingerprint of the entry at source 7 and its kind 1 edge", be, found, err)
|
||||
}
|
||||
behind := func() []meshcatalog.Outdated {
|
||||
t.Helper()
|
||||
a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return a.Behind
|
||||
}
|
||||
if b := behind(); len(b) != 0 {
|
||||
t.Fatalf("behind before lib's next build: %+v", b)
|
||||
}
|
||||
pub("lib", "lib:pkg", "d2", 8)
|
||||
b := behind()
|
||||
if len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Build != fp("b") || b[0].Version != "" || b[0].Dependency.Branch != "beta" ||
|
||||
!strings.Contains(b[0].Recorded, "d1") || !strings.Contains(b[0].Newest, "d2") || b[0].Act != "rebuild" {
|
||||
t.Fatalf("behind after lib's newer build on beta: %+v", b)
|
||||
}
|
||||
if err := ix.PruneBranchBuilds(ctx, "app", []string{"main", "beta"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")); !found {
|
||||
t.Errorf("a branch still watched lost its build-edges")
|
||||
}
|
||||
if err := ix.PruneBranchBuilds(ctx, "app", []string{"main"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")); found {
|
||||
t.Errorf("build-edges of app on beta stayed after beta was unwatched")
|
||||
}
|
||||
if _, _, found, _ := ix.Heard.Get(ctx, BranchBuildKey("app", "beta")); found {
|
||||
t.Errorf("branch-build of app on beta stayed after beta was unwatched")
|
||||
}
|
||||
if _, _, found, _ := ix.Heard.Get(ctx, BranchBuildKey("lib", "beta")); !found {
|
||||
t.Errorf("lib's branch-build went with app's")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// ApplyBranchSource takes a watched branch other than the first in from the source revision the seat answered for it
|
||||
// (design 47 G §13, §14, §291, ADR 0666 §6, A-85, A-93): a pure computation from the merged messages of the heads the source
|
||||
// names, over a view in memory seeded from the branch's newest revision, written straight into the branch's next revision
|
||||
// (KeepBranchEntries): each module's fingerprint, inputs and graph node at the branch's heads, computed as the trunk's are
|
||||
// (applyMoving). Nothing of the trunk is written: its modules, manifests, taken state, graph and packages stay its own,
|
||||
// and the branch has no bucket of its own. A head whose merged message is not read keeps no revision, said; the next
|
||||
// source-moved of the branch takes it in again.
|
||||
func (ix *Indexer) ApplyBranchSource(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error {
|
||||
if isTrunkBranch(branch) || ix.Revisions == nil || ix.Message == nil || len(a.Repositories) == 0 {
|
||||
return nil
|
||||
}
|
||||
heads := map[string]string{}
|
||||
for _, r := range a.Repositories {
|
||||
heads[r.Repository] = r.Head
|
||||
}
|
||||
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: the branch %s is not taken in yet: its objects %s are not on the bus", branch, CatalogRevisionsObjects)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d := ix.dry()
|
||||
// The branch's own view: its modules and graph from its revision, no taken merge, and the packages it publishes in memory
|
||||
// over the trunk's, never written to them.
|
||||
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
|
||||
if ix.Packages != nil {
|
||||
d.ix.Packages = newOverlay(ix.Packages)
|
||||
}
|
||||
d.ix.source = sourceNow{rev: uint64(a.Revision), heads: maps.Clone(heads)}
|
||||
d.ix.Logf = func(format string, args ...any) {
|
||||
ix.logf("branch "+branch+", computed and not written: "+format, args...)
|
||||
}
|
||||
if unread := d.ix.takeInHeads(ctx, heads); len(unread) > 0 {
|
||||
ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: %d repositories were not taken in (%v); taken in again at its next source-moved", branch, a.Revision, len(unread), unread)
|
||||
return nil
|
||||
}
|
||||
entries, err := d.ix.viewEntries(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.KeepBranchEntries(ctx, branch, a.Revision, entries)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
)
|
||||
|
||||
// Design 47 G §13, §14, §291, A-93: a watched branch other than the first is taken in as a pure computation from the merged
|
||||
// messages of its heads, straight into branch.<branch>.<n>; the trunk's modules, graph and revision stay the trunk's.
|
||||
func TestABranchOtherThanTheFirstIsTakenInIntoItsOwnRevisionAndTheTrunkIsLeftAlone(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, mods, merged := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
ix.Revisions = newMemObjects(0)
|
||||
ix.Graph = newMem()
|
||||
ix.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) {
|
||||
if repository != catalogueRepo || head != merged.Merge {
|
||||
return nil, fmt.Errorf("no merge of %s at %s", repository, head)
|
||||
}
|
||||
return json.Marshal(merged)
|
||||
}
|
||||
var said []meshcatalog.CatalogRevisionEvent
|
||||
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
said = append(said, e)
|
||||
return nil
|
||||
}
|
||||
before := map[string]string{}
|
||||
keys, _ := mods.Keys(ctx)
|
||||
for _, k := range keys {
|
||||
v, _, _, _ := mods.Get(ctx, k)
|
||||
before[k] = string(v)
|
||||
}
|
||||
graphBefore, _ := ix.Graph.Keys(ctx)
|
||||
answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}}
|
||||
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || said[0].Revision != 1 || said[0].Branch != "beta" || said[0].Source != 9 || said[0].Entries != int64(len(builderModules(t))) {
|
||||
t.Fatalf("the branch take-in said %+v, want revision 1 of beta at source 9 with every module", said)
|
||||
}
|
||||
rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1))
|
||||
if err != nil || rev.Branch != "beta" || len(entries) != len(builderModules(t)) {
|
||||
t.Fatalf("branch.beta.1 = %+v with %d entries (%v)", rev, len(entries), err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Commit != merged.Merge || e.Inputs == nil { // computed at the branch's head, with the inputs its fingerprint stands on
|
||||
t.Errorf("the entry of %s is at %q with inputs %v, want the branch's head %s with inputs", e.Module, e.Commit, e.Inputs, merged.Merge)
|
||||
}
|
||||
}
|
||||
if _, any, _ := newestTrunk(ctx, ix.Revisions); any {
|
||||
t.Error("a branch take-in kept a trunk revision")
|
||||
}
|
||||
after := map[string]string{}
|
||||
keys, _ = mods.Keys(ctx)
|
||||
for _, k := range keys {
|
||||
v, _, _, _ := mods.Get(ctx, k)
|
||||
after[k] = string(v)
|
||||
}
|
||||
if !maps.Equal(before, after) {
|
||||
t.Errorf("a branch take-in wrote the trunk's modules")
|
||||
}
|
||||
if graphAfter, _ := ix.Graph.Keys(ctx); !slices.Equal(graphBefore, graphAfter) {
|
||||
t.Errorf("a branch take-in wrote the trunk's graph: %v, was %v", graphAfter, graphBefore)
|
||||
}
|
||||
// Taken in again at the same heads, nothing moved: no second revision.
|
||||
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil || len(said) != 1 {
|
||||
t.Errorf("the same heads said %d revisions (%v), want one", len(said), err)
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,17 @@ type Indexer struct {
|
||||
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
|
||||
// id given (W4.6).
|
||||
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
|
||||
// Versions is the state versions (versions.go, design 47 G §349): each module's released versions; nil keeps none.
|
||||
Versions Store
|
||||
// BuildEdges is the state build-edges (branch_builds.go, design 47 G §374): the edge entries of the newest build of a module on a
|
||||
// watched branch other than the first; nil keeps none.
|
||||
BuildEdges Store
|
||||
// SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing.
|
||||
SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error
|
||||
// SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing.
|
||||
SayVersionsFilled func(ctx context.Context, e meshcatalog.VersionsFilledEvent, id string) error
|
||||
// Inputs is the state inputs (run_inputs.go, design 47 G §354): the inputs of the indexer's run asks; nil keeps none.
|
||||
Inputs Store
|
||||
// Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads
|
||||
// what the first registrations are composed from (§8). Nil: no registration act is answered.
|
||||
Registry *Registry
|
||||
@@ -147,26 +158,28 @@ type Indexer struct {
|
||||
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
|
||||
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
freshMu sync.Mutex
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
|
||||
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
freshMu sync.Mutex
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
|
||||
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
||||
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
|
||||
OwnBuild func(ctx context.Context) (string, error)
|
||||
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
||||
@@ -183,7 +196,15 @@ type Indexer struct {
|
||||
ownWhy string
|
||||
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
|
||||
OwnBuildState Store
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
|
||||
// toolchain.<language>, delivered.<module>, pending.<module>.<version>, acted.<acting part>.<dependent>, each key part escaped as a name
|
||||
// (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled.
|
||||
Heard Store
|
||||
// Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the
|
||||
// edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered".
|
||||
Placement PlacementAsker
|
||||
RunningWhole RunningReader
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
// Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a
|
||||
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
|
||||
Message func(ctx context.Context, repository, merge string, number int) ([]byte, error)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// Design 47 G A-85, §291: the indexer keeps a catalog revision per watched branch, trunk.<n> for the first and
|
||||
// branch.<branch>.<n> (the branch escaped) for another, each numbered by its own branch, and the verb revision answers the
|
||||
// branch it is asked for; the event names the branch, escaped.
|
||||
func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Revisions: newMemObjects(1 << 20), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
|
||||
var said []meshcatalog.CatalogRevisionEvent
|
||||
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil }
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Commit: "c1", Invokes: []string{RevisionInvoke}, Manifest: map[string]any{}}
|
||||
tip := func(commit string) map[string]meshcatalog.CatalogEntry {
|
||||
e := reader
|
||||
e.Commit = commit
|
||||
return map[string]meshcatalog.CatalogEntry{"reader": e}
|
||||
}
|
||||
if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, _ := ix.Revisions.Names(ctx)
|
||||
have := map[string]bool{}
|
||||
for _, n := range names {
|
||||
have[n] = true
|
||||
}
|
||||
for _, want := range []string{"trunk.1", "branch.feature/x=2ey.1", "branch.feature/x=2ey.2"} {
|
||||
if !have[want] {
|
||||
t.Errorf("the objects hold %v, want %s", names, want)
|
||||
}
|
||||
}
|
||||
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 {
|
||||
t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said)
|
||||
}
|
||||
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
|
||||
a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
|
||||
if err != nil {
|
||||
t.Fatalf("revision of %q: %v", branch, err)
|
||||
}
|
||||
return a
|
||||
}
|
||||
if a := answer("main", 0); a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" {
|
||||
t.Errorf("main answered %+v", a)
|
||||
}
|
||||
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
|
||||
t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision)
|
||||
}
|
||||
if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
|
||||
t.Errorf("the branch's revision 1 answered %+v", a.Revision)
|
||||
}
|
||||
if a := answer("beta", 0); !a.NoneYet {
|
||||
t.Errorf("a branch with no revision answered %+v, want none-yet", a)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
@@ -46,15 +47,41 @@ type Objects interface {
|
||||
// EntryObject is the object name of the entry of that digest.
|
||||
func EntryObject(digest string) string { return "entry." + digest }
|
||||
|
||||
// RevisionObject is the object name of a revision: trunk.<revision>, or candidate.<group>.<heads> for a candidate,
|
||||
// TrunkBranch is the first watched branch, whose revisions are kept as trunk.<revision> (design 47 G §291, A-85).
|
||||
const TrunkBranch = "main"
|
||||
|
||||
// isTrunkBranch says whether a branch's revisions are the trunk's: the first watched branch, or none named.
|
||||
func isTrunkBranch(branch string) bool { return branch == "" || branch == TrunkBranch }
|
||||
|
||||
// BranchObject is the object name of a revision of a watched branch other than the first: branch.<branch>.<revision>,
|
||||
// the branch escaped as a name (A-85), so a dot in it never reads as a part.
|
||||
func BranchObject(branch string, revision int64) string {
|
||||
return "branch." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
|
||||
}
|
||||
|
||||
// RevisionObject is the object name of a revision: trunk.<revision> for the first watched branch,
|
||||
// branch.<branch>.<revision> for another watched branch (A-85), or candidate.<group>.<heads> for a candidate,
|
||||
// which keeps its newest revision only.
|
||||
func RevisionObject(rev meshcatalog.CatalogRevision) string {
|
||||
if rev.Kind == "candidate" {
|
||||
return "candidate." + rev.Group + "." + rev.Heads
|
||||
}
|
||||
if !isTrunkBranch(rev.Branch) {
|
||||
return BranchObject(rev.Branch, rev.Revision)
|
||||
}
|
||||
return "trunk." + strconv.FormatInt(rev.Revision, 10)
|
||||
}
|
||||
|
||||
// BranchRevisionOf is the revision number a branch object's name holds for that branch, and whether it is one.
|
||||
func BranchRevisionOf(branch, name string) (int64, bool) {
|
||||
n, ok := strings.CutPrefix(name, "branch."+keys.EscapeName(branch)+".")
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
r, err := strconv.ParseInt(n, 10, 64)
|
||||
return r, err == nil
|
||||
}
|
||||
|
||||
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
|
||||
func TrunkRevisionOf(name string) (int64, bool) {
|
||||
n, ok := strings.CutPrefix(name, "trunk.")
|
||||
@@ -128,26 +155,37 @@ func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.Cata
|
||||
|
||||
// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false.
|
||||
func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) {
|
||||
return newestOf(ctx, o, "")
|
||||
}
|
||||
|
||||
// newestOf is the newest revision of a watched branch the bucket keeps (the trunk's for the first watched branch or none
|
||||
// named), its list only; none is a zero revision and false.
|
||||
func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, bool, error) {
|
||||
names, err := o.Names(ctx)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogRevision{}, false, err
|
||||
}
|
||||
number := TrunkRevisionOf
|
||||
if !isTrunkBranch(branch) {
|
||||
number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) }
|
||||
}
|
||||
var newest int64
|
||||
for _, n := range names {
|
||||
if r, ok := TrunkRevisionOf(n); ok && r > newest {
|
||||
if r, ok := number(n); ok && r > newest {
|
||||
newest = r
|
||||
}
|
||||
}
|
||||
if newest == 0 {
|
||||
return meshcatalog.CatalogRevision{}, false, nil
|
||||
}
|
||||
data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10))
|
||||
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest})
|
||||
data, err := o.Get(ctx, name)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogRevision{}, false, err
|
||||
}
|
||||
var rev meshcatalog.CatalogRevision
|
||||
if err := json.Unmarshal(data, &rev); err != nil {
|
||||
return rev, false, fmt.Errorf("trunk.%d: %w", newest, err)
|
||||
return rev, false, fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
return rev, true, nil
|
||||
}
|
||||
@@ -208,15 +246,25 @@ func (ix *Indexer) KeepView(ctx context.Context) error {
|
||||
if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() {
|
||||
return nil
|
||||
}
|
||||
held, err := ix.heldModules(ctx)
|
||||
entries, err := ix.viewEntries(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.keepEntries(ctx, entries)
|
||||
}
|
||||
|
||||
// viewEntries is the entry of every module the indexer's view (its modules and its module graph) holds: the module's value
|
||||
// with its node of the graph, as KeepView keeps them.
|
||||
func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.CatalogEntry, error) {
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := graph.Graph{}
|
||||
if ix.Graph != nil {
|
||||
keys, err := ix.Graph.Keys(ctx)
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
return fmt.Errorf("the module graph: %w", err)
|
||||
return nil, fmt.Errorf("the module graph: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found {
|
||||
@@ -237,7 +285,7 @@ func (ix *Indexer) KeepView(ctx context.Context) error {
|
||||
}
|
||||
entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, m, raw, nodes[name]), m)
|
||||
}
|
||||
return ix.keepEntries(ctx, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry
|
||||
@@ -246,12 +294,26 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
|
||||
if ix.catchingUp.Load() {
|
||||
return nil // the catch-up of a first start keeps its one revision at its end (CatchUp)
|
||||
}
|
||||
ix.keepMu.Lock()
|
||||
defer ix.keepMu.Unlock()
|
||||
ix.mu.Lock()
|
||||
src := int64(ix.source.rev)
|
||||
ix.mu.Unlock()
|
||||
newest, any, err := newestTrunk(ctx, ix.Revisions)
|
||||
return ix.keepBranchEntries(ctx, TrunkBranch, src, entries)
|
||||
}
|
||||
|
||||
// KeepBranchEntries keeps entries as the revision of a watched branch after that branch's newest (design 47 G §291,
|
||||
// A-85): trunk.<revision> for the first watched branch, branch.<branch>.<revision> for another, numbered per branch, and
|
||||
// says catalog-revision naming the branch. src is the branch's source revision the entries were taken in from.
|
||||
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error {
|
||||
return ix.keepBranchEntries(ctx, branch, src, entries)
|
||||
}
|
||||
|
||||
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error {
|
||||
if branch == "" {
|
||||
branch = TrunkBranch
|
||||
}
|
||||
ix.keepMu.Lock()
|
||||
defer ix.keepMu.Unlock()
|
||||
newest, any, err := newestOf(ctx, ix.Revisions, branch)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
|
||||
return nil
|
||||
@@ -261,7 +323,7 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
|
||||
return fmt.Errorf("the newest catalog graph revision: %w", err)
|
||||
}
|
||||
at := ix.now().UTC()
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries)
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -295,7 +357,7 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
|
||||
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
|
||||
if ix.SayCatalogRevision != nil {
|
||||
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
|
||||
Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
|
||||
Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
|
||||
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
|
||||
}
|
||||
}
|
||||
@@ -362,8 +424,14 @@ func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
|
||||
// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's
|
||||
// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view.
|
||||
func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) {
|
||||
return BranchView(ctx, o, TrunkBranch)
|
||||
}
|
||||
|
||||
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
|
||||
// (design 47 G §291, A-93), empty and at 0 when it keeps none.
|
||||
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
|
||||
view, held := newOverlay(nil), newOverlay(nil)
|
||||
newest, any, err := newestTrunk(ctx, o)
|
||||
newest, any, err := newestOf(ctx, o, branch)
|
||||
if err != nil || !any {
|
||||
return view, held, 0, err
|
||||
}
|
||||
|
||||
@@ -908,9 +908,9 @@ func (ix *Indexer) dry() *dryRun {
|
||||
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
||||
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
||||
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
||||
OwnBuildState: nil, underSeats: true,
|
||||
OwnBuildState: nil, Heard: nil, Placement: nil, RunningWhole: nil, underSeats: true,
|
||||
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil,
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil,
|
||||
// A dry run writes no registration and asks no one.
|
||||
Registry: nil, SeedModules: nil}
|
||||
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
||||
|
||||
@@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged",
|
||||
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
|
||||
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules"} {
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges"} {
|
||||
named[f] = true
|
||||
}
|
||||
ty := reflect.TypeOf(Indexer{})
|
||||
|
||||
@@ -122,6 +122,9 @@ type Registry struct {
|
||||
HistoryOf func(ctx context.Context, module string) ([]meshcatalog.Registration, error)
|
||||
// Say says the seat mesh-catalog's event registration-changed.
|
||||
Say func(ctx context.Context, e meshcatalog.RegistrationChangedEvent) error
|
||||
// Pruned is told after a registration changed which branches the module watches, so what is kept of its builds of a branch
|
||||
// it no longer watches goes (registration-changed, design 47 G §374); nil prunes none.
|
||||
Pruned func(ctx context.Context, module string, watched []string) error
|
||||
// Branch reads a branch through the seat git; an empty branch asks the repository's default branch.
|
||||
Branch func(repository, branch string) (gitseat.BranchAnswer, error)
|
||||
// Placement reads a module's placements through the seat mesh-module-assigner's verb placement: one per watched branch a row
|
||||
@@ -816,6 +819,11 @@ func (r *Registry) write(ctx context.Context, key string, p Proposal, next meshc
|
||||
r.logf("registration-changed of %s %d is not said: %v", next.Module, next.Version, err)
|
||||
}
|
||||
}
|
||||
if r.Pruned != nil {
|
||||
if err := r.Pruned(ctx, next.Module, next.Branches); err != nil {
|
||||
r.logf("the builds of %s on the branches it no longer watches are not pruned: %v", next.Module, err)
|
||||
}
|
||||
}
|
||||
return r.endKey(ctx, key, p, ProposalApplied, fmt.Sprintf("applied as version %d of %s on %s", next.Version, next.Module, occ.Says(w)))
|
||||
}
|
||||
|
||||
|
||||
@@ -5,11 +5,11 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
@@ -53,6 +53,22 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
|
||||
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
|
||||
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
|
||||
}
|
||||
// A watched branch other than the first keeps its own revisions (A-85); the first's are the trunk's. The branch is
|
||||
// named as a key name is escaped, so it is read back to the branch's own name.
|
||||
branch, err := keys.UnescapeName(in.Branch)
|
||||
if err != nil && in.Branch != "" {
|
||||
return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err)
|
||||
}
|
||||
if !isTrunkBranch(branch) {
|
||||
other, found, err := newestOf(ctx, o, branch)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err)
|
||||
}
|
||||
if !found {
|
||||
return meshcatalog.RevisionAnswer{NoneYet: true}, nil
|
||||
}
|
||||
newest = other
|
||||
}
|
||||
name := RevisionObject(newest)
|
||||
switch {
|
||||
case in.Group != "" || in.Heads != "":
|
||||
@@ -61,10 +77,14 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
|
||||
}
|
||||
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads})
|
||||
case in.Revision != 0 && in.Revision != newest.Revision:
|
||||
name = "trunk." + strconv.FormatInt(in.Revision, 10)
|
||||
name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: in.Revision})
|
||||
}
|
||||
rev, entries := newest, trunkEntries
|
||||
if name != RevisionObject(newest) {
|
||||
if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" {
|
||||
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
} else if name != RevisionObject(newest) {
|
||||
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The indexer's inputs (design 47 G §354): the input of the ask `run` the indexer's validation puts, kept in the old state
|
||||
// section inputs, one key per input named by its write time and its sha256, read by the seat mesh-catalog's verb input (whose
|
||||
// callers are the holders of the build seats) and by the tool indexer_inputs. A key older than a day is deleted at each new
|
||||
// ask, judged over at most the 20 oldest keys.
|
||||
const (
|
||||
InputsState = "inputs"
|
||||
InputsBucket = Module + "_" + InputsState
|
||||
InputsTool = "indexer_inputs"
|
||||
// InputsKeptFor is how long an input is kept at the longest.
|
||||
InputsKeptFor = 24 * time.Hour
|
||||
// InputsSwept is how many of the oldest keys each new ask judges.
|
||||
InputsSwept = 20
|
||||
inputsTime = "20060102T150405Z"
|
||||
// InputsAskPrefix opens the keys that name an input by the id of the run ask it was kept for (A-78 d): ask.<id>, the id escaped as a
|
||||
// name (§73), holding the input's own key, so the build seat's step-ended of that ask finds it and deletes it.
|
||||
InputsAskPrefix = "ask."
|
||||
)
|
||||
|
||||
// InputDigest is the sha256 of an input's exact bytes, as "sha256:" and its hex digits.
|
||||
func InputDigest(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// KeepInput keeps the exact bytes of the input of the run ask ask under their write time and sha256, after sweeping the keys more
|
||||
// than a day old among the 20 oldest, and names it by the ask's id (A-78 d); it answers the digest. The same bytes written again in
|
||||
// the same second are one key.
|
||||
func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (string, error) {
|
||||
if ask == "" {
|
||||
return "", errors.New("name the run ask the input is kept for")
|
||||
}
|
||||
if ix.Inputs == nil {
|
||||
return "", errors.New("the state inputs is not kept here")
|
||||
}
|
||||
if err := ix.sweepInputs(ctx); err != nil {
|
||||
ix.logf("inputs: the keys older than a day are not all deleted: %v", err)
|
||||
}
|
||||
digest := InputDigest(body)
|
||||
key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest)
|
||||
data, err := json.Marshal(base64.StdEncoding.EncodeToString(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ix.Inputs.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return "", err
|
||||
}
|
||||
if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask
|
||||
// that kept none, or whose input a sweep deleted, changes nothing.
|
||||
func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error {
|
||||
if ix.Inputs == nil || e.Ask == "" {
|
||||
return nil
|
||||
}
|
||||
askKey := InputsAskPrefix + keys.EscapeName(e.Ask)
|
||||
raw, rev, found, err := ix.Inputs.Get(ctx, askKey)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ix.Inputs.Delete(ctx, askKey, rev)
|
||||
}
|
||||
|
||||
// sweepInputs deletes, among the 20 oldest keys, those written more than a day ago.
|
||||
func (ix *Indexer) sweepInputs(ctx context.Context) error {
|
||||
all, err := ix.Inputs.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
var inputs, asks []string
|
||||
for _, k := range all {
|
||||
if strings.HasPrefix(k, InputsAskPrefix) {
|
||||
asks = append(asks, k)
|
||||
} else {
|
||||
inputs = append(inputs, k)
|
||||
}
|
||||
}
|
||||
slices.Sort(inputs) // the write time leads the key, so the oldest come first
|
||||
if len(inputs) > InputsSwept {
|
||||
inputs = inputs[:InputsSwept]
|
||||
}
|
||||
for _, k := range inputs {
|
||||
t, _, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
|
||||
at, perr := time.Parse(inputsTime, t)
|
||||
if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
|
||||
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// a name by ask whose input a sweep or its step-ended deleted goes with it
|
||||
for _, k := range asks {
|
||||
if raw, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
|
||||
if _, _, ok, err := ix.Inputs.Get(ctx, string(raw)); err == nil && !ok {
|
||||
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// InputKey is the key of the input of the digest, or none.
|
||||
func InputKey(ctx context.Context, st Store, digest string) (string, error) {
|
||||
all, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, k := range all {
|
||||
if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest && !strings.HasPrefix(k, InputsAskPrefix) {
|
||||
return k, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Input answers the verb input: the exact bytes (base64) of the input of the digest, its digest checked.
|
||||
func Input(ctx context.Context, st Store, in meshcatalog.InputInput) (meshcatalog.InputAnswer, error) {
|
||||
var out meshcatalog.InputAnswer
|
||||
if in.Digest == "" {
|
||||
return out, errors.New("name the input's sha256 (digest)")
|
||||
}
|
||||
key, err := InputKey(ctx, st, in.Digest)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the state %s cannot be read: %w", InputsState, err)
|
||||
}
|
||||
raw, _, found, err := st.Get(ctx, key)
|
||||
if key == "" || err != nil || !found {
|
||||
return out, fmt.Errorf("no input of digest %s is kept", in.Digest)
|
||||
}
|
||||
text, err := meshcatalog.DecodeInputsByTimeDigestValue(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the input %s cannot be read: %w", key, err)
|
||||
}
|
||||
body, err := base64.StdEncoding.DecodeString(text)
|
||||
if err != nil || InputDigest(body) != in.Digest {
|
||||
return out, fmt.Errorf("the input kept under %s does not match its digest", key)
|
||||
}
|
||||
return meshcatalog.InputAnswer{Digest: in.Digest, Input: text}, nil
|
||||
}
|
||||
|
||||
// InputTool is the verb input as the tool mesh-catalog.input of the indexer's own runtime.
|
||||
func InputTool(st Store) stdio.Tool {
|
||||
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbInput, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.InputInput
|
||||
if raw, err := json.Marshal(args); err == nil {
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
}
|
||||
return Input(context.Background(), st, in)
|
||||
}}
|
||||
}
|
||||
|
||||
// InputsAnswer is what indexer_inputs answers: each key kept with its time, digest and size, or one input's bytes.
|
||||
type InputsAnswer struct {
|
||||
Inputs []InputKept `json:"inputs"`
|
||||
}
|
||||
|
||||
// InputKept is one input kept.
|
||||
type InputKept struct {
|
||||
Key string `json:"key"`
|
||||
Digest string `json:"digest"`
|
||||
Bytes int `json:"bytes"`
|
||||
Input string `json:"input,omitempty"`
|
||||
}
|
||||
|
||||
// InputsReadTool is indexer_inputs (class read): every input the indexer keeps, with its key, digest and size; with a digest,
|
||||
// that input whole (base64).
|
||||
func InputsReadTool(st Store) stdio.Tool {
|
||||
return stdio.Tool{Name: InputsTool, Description: "Read the indexer's inputs of the ask run (design 47 G section 354): each key (write time and sha256) with its size; with a digest, that input's exact bytes, base64.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"digest": map[string]any{"type": "string", "description": "one input's sha256; none lists every key"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ctx := context.Background()
|
||||
want, _ := args["digest"].(string)
|
||||
all, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(all)
|
||||
out := InputsAnswer{Inputs: []InputKept{}}
|
||||
for _, k := range all {
|
||||
_, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
|
||||
if !ok || strings.HasPrefix(k, InputsAskPrefix) || (want != "" && d != want) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := st.Get(ctx, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
text, _ := meshcatalog.DecodeInputsByTimeDigestValue(raw)
|
||||
kept := InputKept{Key: k, Digest: d, Bytes: base64.StdEncoding.DecodedLen(len(text))}
|
||||
if want != "" {
|
||||
kept.Input = text
|
||||
}
|
||||
out.Inputs = append(out.Inputs, kept)
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// StepsVerb is the verb of each build seat the seed asks (design 47 G §704).
|
||||
const StepsVerb = "steps"
|
||||
|
||||
// StepsCaller asks the build seat seat its verb steps (a kept call, ADR 0668 §1): the answer, or why none came.
|
||||
type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error)
|
||||
|
||||
// ModuleManifest is the file name of a module's manifest in its directory.
|
||||
const ModuleManifest = "module.json"
|
||||
|
||||
// buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704,
|
||||
// §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said.
|
||||
func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) {
|
||||
var seats []string
|
||||
for _, name := range slices.Sorted(maps.Keys(defs)) {
|
||||
for _, r := range defs[name].Acts {
|
||||
if r.Step != seatdef.StepBuild {
|
||||
continue
|
||||
}
|
||||
if r.ActsOn(a.Language) || (r.On == seatdef.OnKindImage && a.Kind == "image") {
|
||||
seats = append(seats, name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
switch len(seats) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("no seat builds the artifact %s (language %q, kind %q)", a.Ref, a.Language, a.Kind)
|
||||
case 1:
|
||||
return seats[0], nil
|
||||
}
|
||||
return "", fmt.Errorf("the seats %v each build the artifact %s", seats, a.Ref)
|
||||
}
|
||||
|
||||
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
|
||||
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
|
||||
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
|
||||
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
|
||||
// it is unanswered five minutes and cleared when it answers.
|
||||
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
|
||||
if ix.Revisions == nil {
|
||||
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
|
||||
for _, e := range entries {
|
||||
for _, a := range e.Artifacts {
|
||||
seat, err := buildSeatOf(defs, a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", e.Module, err)
|
||||
}
|
||||
seatOf[a.Ref] = seat
|
||||
}
|
||||
}
|
||||
answered := map[string]bool{}
|
||||
unanswered := map[string]error{}
|
||||
var out []SeedModule
|
||||
for _, e := range entries {
|
||||
m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)}
|
||||
for _, a := range e.Artifacts {
|
||||
seat := seatOf[a.Ref]
|
||||
name := artifactName(a.Ref)
|
||||
sa := SeedArtifact{Name: name, Seat: seat}
|
||||
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
|
||||
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name), Packages: maps.Clone(e.Packages)}}
|
||||
ans, err := steps(ctx, seat, in)
|
||||
switch {
|
||||
case err != nil:
|
||||
unanswered[seat] = err
|
||||
case ans.Error != "":
|
||||
unanswered[seat] = fmt.Errorf("%s", ans.Error)
|
||||
default:
|
||||
sa.Answered = true
|
||||
answered[seat] = true
|
||||
for _, s := range ans.Steps {
|
||||
sa.Steps = append(sa.Steps, s.Step)
|
||||
}
|
||||
}
|
||||
m.Artifacts = append(m.Artifacts, sa)
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
ix.stepsFindings(ctx, answered, unanswered)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// artifactName is the artifact's name in an artifact reference <module>:<artifact>.
|
||||
func artifactName(ref string) string {
|
||||
for i := len(ref) - 1; i >= 0; i-- {
|
||||
if ref[i] == ':' {
|
||||
return ref[i+1:]
|
||||
}
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
|
||||
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
now := ix.now()
|
||||
var raise []string
|
||||
since := map[string]time.Time{}
|
||||
for seat := range unanswered {
|
||||
since[seat] = ix.stepsUnansweredSince(ctx, seat, now)
|
||||
if now.Sub(since[seat]) >= mcc.CallUnansweredAfter {
|
||||
raise = append(raise, seat)
|
||||
}
|
||||
}
|
||||
for seat := range answered {
|
||||
if _, bad := unanswered[seat]; !bad {
|
||||
ix.stepsAnswered(ctx, seat)
|
||||
}
|
||||
}
|
||||
for _, seat := range slices.Sorted(slices.Values(raise)) {
|
||||
id := seat + "." + StepsVerb
|
||||
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("seed: the finding %s is not written: %v", id, err)
|
||||
}
|
||||
}
|
||||
for seat := range answered {
|
||||
if _, bad := unanswered[seat]; bad {
|
||||
continue
|
||||
}
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
|
||||
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// namedPaths are the named paths the entry keeps for the artifact, by its name or its reference.
|
||||
func namedPaths(e meshcatalog.CatalogEntry, name string) []string {
|
||||
if e.Inputs == nil {
|
||||
return nil
|
||||
}
|
||||
if p, ok := e.Inputs.Paths[name]; ok {
|
||||
return p
|
||||
}
|
||||
return e.Inputs.Paths[ArtifactRefOf(e.Module, name)]
|
||||
}
|
||||
|
||||
// StepsSincePrefix opens the keys of the state heard (Heard, A-80) that keep, by build seat, since when its steps
|
||||
// has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past
|
||||
// five minutes.
|
||||
const StepsSincePrefix = "steps-since."
|
||||
|
||||
// stepsUnansweredSince is the time the seat first failed to answer.
|
||||
func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time {
|
||||
return ix.unansweredSince(ctx, StepsSincePrefix+keys.EscapeName(seat), now)
|
||||
}
|
||||
|
||||
// stepsAnswered forgets the time a seat that answers was unanswered since.
|
||||
func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) {
|
||||
ix.answered(ctx, StepsSincePrefix+keys.EscapeName(seat))
|
||||
}
|
||||
|
||||
// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's state heard (the first
|
||||
// failure writes now); where no state is kept it is held in memory.
|
||||
func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time {
|
||||
if ix.Heard == nil {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if ix.stepsSince == nil {
|
||||
ix.stepsSince = map[string]time.Time{}
|
||||
}
|
||||
if _, ok := ix.stepsSince[key]; !ok {
|
||||
ix.stepsSince[key] = now
|
||||
}
|
||||
return ix.stepsSince[key]
|
||||
}
|
||||
if v, _, found, err := ix.Heard.Get(ctx, key); err == nil && found {
|
||||
if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
if err := ix.Heard.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil {
|
||||
ix.logf("the time since %s is unanswered is not kept: %v", key, err)
|
||||
}
|
||||
return now
|
||||
}
|
||||
|
||||
// answered forgets the time the call kept at key was unanswered since.
|
||||
func (ix *Indexer) answered(ctx context.Context, key string) {
|
||||
if ix.Heard == nil {
|
||||
ix.mu.Lock()
|
||||
delete(ix.stepsSince, key)
|
||||
ix.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, key); err == nil && found {
|
||||
if err := ix.Heard.Delete(ctx, key, rev); err != nil {
|
||||
ix.logf("the time since %s is unanswered is not forgotten: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// Design 47 G §704: the seed asks `steps` of the build seat whose `build` rule matches each artifact, with {module,
|
||||
// artifact, fingerprint, fetch} from the newest trunk entry; a seat that does not answer holds the seed and, after five
|
||||
// minutes, is the finding call-unanswered (scope call, id <seat>.steps), cleared when it answers.
|
||||
func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
w := newFindings()
|
||||
ix := &Indexer{Findings: w, Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }}
|
||||
fp := "src2:" + strings.Repeat("a", 64)
|
||||
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}},
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"},
|
||||
Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defs := map[string]seatdef.Definition{
|
||||
"node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}},
|
||||
"node-build-image": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: seatdef.OnKindImage}}},
|
||||
}
|
||||
// the image artifact is built by both rules: the language's and the kind's
|
||||
if _, err := ix.SeedModulesFrom(ctx, defs, nil); err == nil || !strings.Contains(err.Error(), "each build") {
|
||||
t.Fatalf("two seats building one artifact: %v", err)
|
||||
}
|
||||
e := entries["avahi"]
|
||||
e.Artifacts[1].Language = "dockerfile"
|
||||
entries["avahi"] = e
|
||||
rev, objects, _ = CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 4, Kind: "trunk"}, entries)
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []nodebuildgo.StepsInput
|
||||
goUp := false
|
||||
call := func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) {
|
||||
got = append(got, in)
|
||||
if seat == "node-build-go" && !goUp {
|
||||
return nodebuildgo.StepsAnswer{}, errors.New("no responders")
|
||||
}
|
||||
return nodebuildgo.StepsAnswer{Steps: []nodebuildgo.DetectedStep{{Step: "unit"}}}, nil
|
||||
}
|
||||
mods, err := ix.SeedModulesFrom(ctx, defs, call)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mods) != 1 || mods[0].Module != "avahi" || mods[0].Origin != "novox/r" || mods[0].Path != "mods/avahi/module.json" || len(mods[0].Artifacts) != 2 {
|
||||
t.Fatalf("the seed's modules: %+v", mods)
|
||||
}
|
||||
if a := mods[0].Artifacts[0]; a.Answered || a.Seat != "node-build-go" {
|
||||
t.Errorf("the go artifact is %+v, want unanswered by node-build-go", a)
|
||||
}
|
||||
if a := mods[0].Artifacts[1]; !a.Answered || a.Seat != "node-build-image" || len(a.Steps) != 1 {
|
||||
t.Errorf("the image artifact is %+v", a)
|
||||
}
|
||||
in := got[0]
|
||||
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" {
|
||||
t.Errorf("the steps input is %+v", in)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Fatal("a call unanswered for no time wrote its finding")
|
||||
}
|
||||
now = now.Add(5 * time.Minute)
|
||||
ix = &Indexer{Findings: w, Heard: ix.Heard, Revisions: ix.Revisions, Now: func() time.Time { return now }} // a restart keeps the time
|
||||
if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != mcc.KindCallUnanswered {
|
||||
t.Fatalf("the findings after five minutes: %+v", h)
|
||||
}
|
||||
goUp = true
|
||||
if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Errorf("the finding stands after the seat answered: %+v", w.Held())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
|
||||
// released versions in the state versions, one key per module and version (<module>.<version>) holding each artifact's digest,
|
||||
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
|
||||
// released once every artifact the module's entry names is published at it.
|
||||
const (
|
||||
VersionsState = "versions"
|
||||
VersionsBucket = Module + "_" + VersionsState
|
||||
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
|
||||
// go, never its highest of any MAJOR.MINOR.
|
||||
VersionsKept = 100
|
||||
// PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the
|
||||
// artifacts published at it so far, each key part escaped as a name (A-80, §73).
|
||||
|
||||
PendingPrefix = "pending."
|
||||
)
|
||||
|
||||
// published is what a not yet released version holds: each artifact published at it, by reference.
|
||||
type published struct {
|
||||
Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"`
|
||||
Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"`
|
||||
Source int64 `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// versionParts is MAJOR.MINOR.PATCH, read; false for any other form.
|
||||
func versionParts(v string) ([3]int, bool) {
|
||||
var out [3]int
|
||||
parts := strings.Split(v, ".")
|
||||
if len(parts) != 3 {
|
||||
return out, false
|
||||
}
|
||||
for i, p := range parts {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n < 0 {
|
||||
return out, false
|
||||
}
|
||||
out[i] = n
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// compareVersions orders two versions MAJOR.MINOR.PATCH numerically.
|
||||
func compareVersions(a, b string) int {
|
||||
x, _ := versionParts(a)
|
||||
y, _ := versionParts(b)
|
||||
for i := range x {
|
||||
if x[i] != y[i] {
|
||||
return x[i] - y[i]
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ModuleVersions reads every released version the state keeps for module, newest first.
|
||||
func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) {
|
||||
all, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []meshcatalog.VersionEntry
|
||||
for _, k := range all {
|
||||
m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
|
||||
if !ok || m != module {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := st.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its
|
||||
// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded,
|
||||
// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A
|
||||
// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83).
|
||||
func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error {
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return errors.New("the state versions is not kept here")
|
||||
}
|
||||
if _, ok := versionParts(p.Version); !ok {
|
||||
return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version)
|
||||
}
|
||||
if ix.Registry != nil {
|
||||
if r, found, err := ix.Registry.Registration(ctx, p.Module); err == nil && found && len(r.Branches) > 0 && r.Branches[0] != p.Branch {
|
||||
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
|
||||
}
|
||||
}
|
||||
if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil || found {
|
||||
return err // released already: heard again
|
||||
}
|
||||
entry, err := ix.entryOf(ctx, p.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var want []string
|
||||
for _, a := range entry.Artifacts {
|
||||
want = append(want, a.Ref)
|
||||
}
|
||||
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
||||
var have published
|
||||
err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) {
|
||||
have = published{}
|
||||
if found {
|
||||
if err := json.Unmarshal(raw, &have); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
if have.Artifacts == nil {
|
||||
have.Artifacts = map[string]meshcatalog.VersionArtifact{}
|
||||
}
|
||||
have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest}
|
||||
have.Source = p.Source
|
||||
for _, d := range p.Dependencies {
|
||||
if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) {
|
||||
have.Dependencies = append(have.Dependencies, d)
|
||||
}
|
||||
}
|
||||
data, err := json.Marshal(have)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ref := range want {
|
||||
if _, ok := have.Artifacts[ref]; !ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return ix.releaseVersion(ctx, p, key, entry, have)
|
||||
}
|
||||
|
||||
// entryOf is the module's entry in the newest trunk revision.
|
||||
func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) {
|
||||
if ix.Revisions == nil {
|
||||
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogEntry{}, err
|
||||
}
|
||||
if !found {
|
||||
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return meshcatalog.CatalogEntry{}, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module == module {
|
||||
return e, nil
|
||||
}
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module)
|
||||
}
|
||||
|
||||
// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions,
|
||||
// and forgets the artifacts kept as published.
|
||||
func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error {
|
||||
key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, p.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
||||
at := ix.now().UTC()
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
||||
for _, a := range entry.Artifacts {
|
||||
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
||||
}
|
||||
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
||||
v.Edges = releaseEdges(entry, have.Dependencies)
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return err
|
||||
}
|
||||
if !late && ix.SayVersionReleased != nil {
|
||||
ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at}
|
||||
if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil {
|
||||
return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err)
|
||||
}
|
||||
}
|
||||
if err := ix.trimVersions(ctx, p.Module); err != nil {
|
||||
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
|
||||
}
|
||||
return ix.forgetPublished(ctx, pendingKey)
|
||||
}
|
||||
|
||||
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
||||
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
||||
if err != nil || len(kept) <= VersionsKept {
|
||||
return err
|
||||
}
|
||||
highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first
|
||||
seen := map[string]bool{}
|
||||
for _, v := range kept {
|
||||
p := strings.Split(v.Version, ".")
|
||||
if mm := p[0] + "." + p[1]; !seen[mm] {
|
||||
seen[mm] = true
|
||||
highest[v.Version] = true
|
||||
}
|
||||
}
|
||||
excess := len(kept) - VersionsKept
|
||||
for i := len(kept) - 1; i >= 0 && excess > 0; i-- {
|
||||
v := kept[i].Version
|
||||
if highest[v] {
|
||||
continue
|
||||
}
|
||||
key := meshcatalog.VersionsByModuleVersionKey(module, v)
|
||||
if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found {
|
||||
if err := ix.Versions.Delete(ctx, key, rev); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
excess--
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgetPublished removes the artifacts kept as published at a version released.
|
||||
func (ix *Indexer) forgetPublished(ctx context.Context, key string) error {
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found {
|
||||
return err
|
||||
} else {
|
||||
return ix.Heard.Delete(ctx, key, rev)
|
||||
}
|
||||
}
|
||||
|
||||
// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first.
|
||||
func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) {
|
||||
if in.Module == "" {
|
||||
return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)")
|
||||
}
|
||||
v, err := ModuleVersions(ctx, st, in.Module)
|
||||
if v == nil {
|
||||
v = []meshcatalog.VersionEntry{}
|
||||
}
|
||||
return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err
|
||||
}
|
||||
|
||||
// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime.
|
||||
func VersionsTool(st Store) stdio.Tool {
|
||||
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.VersionsInput
|
||||
if raw, err := json.Marshal(args); err == nil {
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
}
|
||||
return Versions(context.Background(), st, in)
|
||||
}}
|
||||
}
|
||||
|
||||
// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374).
|
||||
const notYetDelivered = "not yet delivered"
|
||||
|
||||
// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build
|
||||
// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's
|
||||
// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here.
|
||||
func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge {
|
||||
word := func(kind int64, part string) string {
|
||||
for _, e := range entry.Edges {
|
||||
if e.Kind == kind && (e.Part == part || e.From == part) {
|
||||
return e.Word
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
var out []meshcatalog.VersionEdge
|
||||
for _, d := range deps {
|
||||
out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)})
|
||||
}
|
||||
for _, e := range entry.Edges {
|
||||
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
|
||||
continue
|
||||
}
|
||||
part := cmp.Or(e.Part, e.From)
|
||||
if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) {
|
||||
out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions
|
||||
// state and the heard toolchain and delivered keys of the own state.
|
||||
func OutdatedTool(ix *Indexer) stdio.Tool {
|
||||
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.OutdatedInput
|
||||
if raw, err := json.Marshal(args); err == nil {
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
}
|
||||
return ix.Outdated(context.Background(), in)
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// What outdated compares against is kept as heard, in the indexer's state heard (design 47 G A-68): toolchain.<language> the newest
|
||||
// toolchain-agreed per language, delivered.<module> the newest version-delivered per module.
|
||||
const (
|
||||
ToolchainPrefix = "toolchain."
|
||||
DeliveredPrefix = "delivered."
|
||||
// ActedPrefix opens acted.<acting part>.<dependent>, the newest wait-ended of an acting part's wait for a dependent, ran or released
|
||||
// (A-78 a), each key part escaped as a name (A-80, §73).
|
||||
ActedPrefix = "acted."
|
||||
)
|
||||
|
||||
// Acted is what an acted key holds: the build the dependent was at when the acting part acted for it (the wait's version, §374 kind 5).
|
||||
type Acted struct {
|
||||
V int64 `json:"v"`
|
||||
Part string `json:"part"`
|
||||
Step string `json:"step"`
|
||||
Build string `json:"build"`
|
||||
How string `json:"how"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HearWaitEnded keeps, for kind 5 (§374, §375, A-78 a), the newest wait-ended of an acting step that ended ran or released: the
|
||||
// acting part, the step and the build it acted for. A wait that ended superseded, or of a step that is no acting part's (put,
|
||||
// delivery, kept-build, backup), is not an act and keeps nothing.
|
||||
func (ix *Indexer) HearWaitEnded(ctx context.Context, w meshwants.WaitEndedEvent) error {
|
||||
if w.Part == "" || w.Module == "" || (w.How != "ran" && w.How != "released") {
|
||||
return nil
|
||||
}
|
||||
switch w.Step {
|
||||
case meshwants.WaitStepPut, meshwants.WaitStepDelivery, meshwants.WaitStepKeptBuild, meshwants.WaitStepBackup, "":
|
||||
return nil
|
||||
}
|
||||
if ix.Heard == nil {
|
||||
return errors.New("the state heard is not kept here")
|
||||
}
|
||||
a := Acted{V: 1, Part: w.Part, Step: w.Step, Build: w.Version, How: w.How, At: w.Since}
|
||||
return ix.keepNewest(ctx, ActedKey(w.Part, w.Module), a, func(kept []byte) bool {
|
||||
var k Acted
|
||||
return json.Unmarshal(kept, &k) != nil || !a.At.Before(k.At)
|
||||
})
|
||||
}
|
||||
|
||||
// ActedKey is the key of the state heard of an acting part's act for a dependent.
|
||||
func ActedKey(part, dependent string) string {
|
||||
return ActedPrefix + keys.EscapeName(part) + "." + keys.EscapeName(dependent)
|
||||
}
|
||||
|
||||
// actedFor is the act kept for the acting part (an artifact reference, or its module) and the dependent module.
|
||||
func (ix *Indexer) actedFor(ctx context.Context, part, dependent string) (Acted, bool) {
|
||||
for _, p := range []string{part, strings.SplitN(part, ":", 2)[0]} {
|
||||
raw, _, found, err := ix.Heard.Get(ctx, ActedKey(p, dependent))
|
||||
var a Acted
|
||||
if err == nil && found && json.Unmarshal(raw, &a) == nil {
|
||||
return a, true
|
||||
}
|
||||
}
|
||||
return Acted{}, false
|
||||
}
|
||||
|
||||
// keepNewest writes body at key of the state heard unless it holds a value whose version is higher; version of the body
|
||||
// is read by newer, which says whether the heard one is not older than the kept.
|
||||
func (ix *Indexer) keepNewest(ctx context.Context, key string, body any, newer func(kept []byte) bool) error {
|
||||
if ix.Heard == nil {
|
||||
return errors.New("the own state is not kept here")
|
||||
}
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx, ix.Heard, key, func(kept []byte, found bool) ([]byte, bool, error) {
|
||||
if found && !newer(kept) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return data, false, nil
|
||||
})
|
||||
}
|
||||
|
||||
// HearToolchainAgreed keeps the newest toolchain-agreed of a language (A-68, §375): the one heard last.
|
||||
func (ix *Indexer) HearToolchainAgreed(ctx context.Context, t meshbuilds.ToolchainAgreed) error {
|
||||
if t.Language == "" {
|
||||
return errors.New("a toolchain-agreed names no language")
|
||||
}
|
||||
return ix.keepNewest(ctx, ToolchainPrefix+keys.EscapeName(t.Language), t, func([]byte) bool { return true })
|
||||
}
|
||||
|
||||
// HearVersionDelivered takes one version-delivered (design 47 G §374, §376, A-68): the entries of kinds 2, 4 and 6 of the
|
||||
// delivered release, which read "not yet delivered", are written once with each dependency's newest delivered version and
|
||||
// part digest; then the delivered version is kept as the module's newest.
|
||||
func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.VersionDeliveredEvent) error {
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return errors.New("the state versions is not kept here")
|
||||
}
|
||||
if _, ok := versionParts(d.Version); !ok {
|
||||
return fmt.Errorf("the version-delivered of %s names the version %q, which is no MAJOR.MINOR.PATCH", d.Module, d.Version)
|
||||
}
|
||||
key := meshcatalog.VersionsByModuleVersionKey(d.Module, d.Version)
|
||||
err := write(ctx, ix.Versions, key, func(raw []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil // a version the catalog does not hold (a build of another branch) has no entries to write
|
||||
}
|
||||
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
changed := false
|
||||
for i, e := range v.Edges {
|
||||
if e.Version != notYetDelivered {
|
||||
continue
|
||||
}
|
||||
dep, ok := ix.deliveredOf(ctx, strings.SplitN(e.Part, ":", 2)[0])
|
||||
if !ok {
|
||||
continue // the dependency is not delivered yet: the entry reads so until this release's own delivery finds it
|
||||
}
|
||||
v.Edges[i].Version, v.Edges[i].Digest = dep.Version, dep.Parts[e.Part]
|
||||
changed = true
|
||||
}
|
||||
if !changed {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := json.Marshal(v)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(d.Module), d, func(kept []byte) bool {
|
||||
var k meshwants.VersionDeliveredEvent
|
||||
return json.Unmarshal(kept, &k) != nil || compareVersions(d.Version, k.Version) >= 0
|
||||
})
|
||||
}
|
||||
|
||||
// deliveredOf is the newest version-delivered kept of a module.
|
||||
func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.VersionDeliveredEvent, bool) {
|
||||
var d meshwants.VersionDeliveredEvent
|
||||
raw, _, found, err := ix.Heard.Get(ctx, DeliveredPrefix+keys.EscapeName(module))
|
||||
if err != nil || !found || json.Unmarshal(raw, &d) != nil {
|
||||
return d, false
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
|
||||
// Outdated answers the verb outdated (design 47 G §375) for every edge kind: every entry of each module's newest released
|
||||
// version that is behind. Kind 1: the recorded digest differs from the part's digest in the dependency's newest released
|
||||
// version. Kind 3: the recorded toolchain version differs from the version of the newest toolchain-agreed of the dependent
|
||||
// artifact's language (A-80, toolchain.<language>). Kinds 2 and 4: from the dependency's newest version-delivered. Kind 5: the
|
||||
// acting part has not acted for the dependent's current build (acted.<acting part>.<dependent> against the newest trunk
|
||||
// entry's fingerprint, A-78 a). Kind 6: the delivered version's catalog entry no longer serves the verb or word (A-78 b). A
|
||||
// stale edge (kinds 1, 3) has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that
|
||||
// left the digest as it was makes nothing behind.
|
||||
func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (meshcatalog.OutdatedAnswer, error) {
|
||||
out := meshcatalog.OutdatedAnswer{Behind: []meshcatalog.Outdated{}}
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return out, errors.New("the state versions is not kept here")
|
||||
}
|
||||
all, err := ix.Versions.Keys(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
modules := map[string]bool{}
|
||||
for _, k := range all {
|
||||
if m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k); ok {
|
||||
modules[m] = true
|
||||
}
|
||||
}
|
||||
newest := map[string]meshcatalog.VersionEntry{}
|
||||
for m := range modules {
|
||||
vs, err := ModuleVersions(ctx, ix.Versions, m)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, v := range vs {
|
||||
if !v.Late {
|
||||
newest[m] = v
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them
|
||||
if ix.Revisions != nil {
|
||||
if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found {
|
||||
if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil {
|
||||
for _, e := range entries {
|
||||
trunk[e.Module] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range sortedKeys(newest) {
|
||||
v := newest[m]
|
||||
for _, e := range v.Edges {
|
||||
dep := strings.SplitN(e.Part, ":", 2)[0]
|
||||
if in.Dependency != "" && in.Dependency != e.Part {
|
||||
continue
|
||||
}
|
||||
o := meshcatalog.Outdated{Module: m, Version: v.Version, Kind: e.Kind,
|
||||
Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: ix.firstBranch(ctx, dep)}}
|
||||
switch e.Kind {
|
||||
case 1:
|
||||
n, ok := newest[dep]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var digest string
|
||||
for _, a := range n.Artifacts {
|
||||
if a.Artifact == e.Part {
|
||||
digest = a.Digest
|
||||
}
|
||||
}
|
||||
if digest == "" || digest == e.Digest {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Digest, n.Version+" "+digest
|
||||
case 3:
|
||||
tc, ok := ix.agreedToolchain(ctx, languageOf(trunk[m], e))
|
||||
if !ok || tc.Version == "" || tc.Version == e.Version {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Digest, tc.Version
|
||||
case 2, 4:
|
||||
d, ok := ix.deliveredOf(ctx, dep)
|
||||
if !ok || e.Version == notYetDelivered {
|
||||
continue
|
||||
}
|
||||
digest := d.Parts[e.Part]
|
||||
if digest == "" || digest == e.Digest {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Digest, d.Version+" "+digest
|
||||
case 5:
|
||||
cur := trunk[m].Fp
|
||||
if cur == "" {
|
||||
continue
|
||||
}
|
||||
acted, ok := ix.actedFor(ctx, e.Part, m)
|
||||
if ok && acted.Build == cur {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = "acted for "+cmp.Or(acted.Build, "no build"), "the current build "+cur
|
||||
case 6:
|
||||
d, ok := ix.deliveredOf(ctx, dep)
|
||||
if !ok || e.Version == notYetDelivered || d.Version == e.Version {
|
||||
continue
|
||||
}
|
||||
if serves, known := ix.deliveredServes(ctx, dep, d.Version, e.Word); !known || serves {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Word, d.Version+" no longer serves "+e.Word
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if e.Stale {
|
||||
o.Act = "rebuild"
|
||||
}
|
||||
out.Behind = append(out.Behind, o)
|
||||
}
|
||||
}
|
||||
behind, err := ix.outdatedBranchBuilds(ctx, in)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Behind = append(out.Behind, behind...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// agreedToolchain is the newest toolchain-agreed kept for a language (A-80).
|
||||
func (ix *Indexer) agreedToolchain(ctx context.Context, language string) (meshbuilds.ToolchainAgreed, bool) {
|
||||
var t meshbuilds.ToolchainAgreed
|
||||
if language == "" {
|
||||
return t, false
|
||||
}
|
||||
raw, _, found, err := ix.Heard.Get(ctx, ToolchainPrefix+keys.EscapeName(language))
|
||||
if err != nil || !found || json.Unmarshal(raw, &t) != nil {
|
||||
return t, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
// languageOf is the language of the artifact a kind-3 edge of the entry builds: the target of the catalog graph's edge from the
|
||||
// toolchain part, else the one language the entry's artifacts share.
|
||||
func languageOf(entry meshcatalog.CatalogEntry, e meshcatalog.VersionEdge) string {
|
||||
lang := map[string]string{}
|
||||
for _, a := range entry.Artifacts {
|
||||
lang[a.Ref] = a.Language
|
||||
}
|
||||
for _, g := range entry.Edges {
|
||||
if g.Kind == 3 && (g.Part == e.Part || g.From == e.Part) && lang[g.To] != "" {
|
||||
return lang[g.To]
|
||||
}
|
||||
}
|
||||
one := ""
|
||||
for _, l := range lang {
|
||||
if l == "" || (one != "" && one != l) {
|
||||
return ""
|
||||
}
|
||||
one = l
|
||||
}
|
||||
return one
|
||||
}
|
||||
|
||||
// deliveredServes says whether the catalog entry of a delivered version of module serves word, its verb (seat.verb, as the entry's
|
||||
// serves lists it, the form `seat:` allowed) or a word of its manifest (A-78 b); known is false where that version's entry cannot be read.
|
||||
func (ix *Indexer) deliveredServes(ctx context.Context, module, version, word string) (serves, known bool) {
|
||||
if ix.Revisions == nil {
|
||||
return false, false
|
||||
}
|
||||
vs, err := ModuleVersions(ctx, ix.Versions, module)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
var source int64
|
||||
for _, v := range vs {
|
||||
if v.Version == version {
|
||||
source = v.Source
|
||||
}
|
||||
}
|
||||
if source == 0 {
|
||||
return false, false
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
var numbers []int64
|
||||
for _, n := range names {
|
||||
if r, ok := TrunkRevisionOf(n); ok {
|
||||
numbers = append(numbers, r)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) })
|
||||
for _, n := range numbers { // the newest revision taken from the source revision the version was built at
|
||||
rev, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", n))
|
||||
if err != nil || rev.Source != source {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module != module {
|
||||
continue
|
||||
}
|
||||
w := strings.TrimPrefix(word, "seat:")
|
||||
_, inManifest := e.Manifest[word]
|
||||
return slices.Contains(e.Serves, w) || slices.Contains(e.Provides, w) || slices.Contains(e.Streams, w) || inManifest, true
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// firstBranch is the first watched branch of a module's registration, which its released versions are of.
|
||||
func (ix *Indexer) firstBranch(ctx context.Context, module string) string {
|
||||
if ix.Registry != nil {
|
||||
if r, found, err := ix.Registry.Registration(ctx, module); err == nil && found && len(r.Branches) > 0 {
|
||||
return r.Branches[0]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The first fill of the versions (design 47 G §372): at the first start under G the state versions is filled once, as
|
||||
// released, for each module from the highest published version over every artifact of the module that the builder's state
|
||||
// versions held, read by the builder's tool builder_versions as a kept call; then versions-filled is said once, and the
|
||||
// builder reserves nothing before it (§369).
|
||||
const (
|
||||
// BuilderVersionsCall is the tool the manifest names in its invokes (until that tool goes).
|
||||
BuilderVersionsCall = "mesh-delivery-builder.builder_versions"
|
||||
// VersionsFilledKey is the key of the state heard that says the fill is done, so it is not made again.
|
||||
VersionsFilledKey = "versions-filled"
|
||||
)
|
||||
|
||||
// versionsFillSince is the key of the state heard of the time builder_versions is first unanswered.
|
||||
var versionsFillSince = "call-since." + keys.EscapeName(BuilderVersionsCall)
|
||||
|
||||
// BuilderVersion is one version of a name in the answer of builder_versions (its fields the fill reads).
|
||||
type BuilderVersion struct {
|
||||
Version string `json:"version"`
|
||||
State string `json:"state"`
|
||||
Digest string `json:"digest"`
|
||||
Revision string `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// BuilderVersionsAnswer is what builder_versions answers for a name.
|
||||
type BuilderVersionsAnswer struct {
|
||||
Name string `json:"name"`
|
||||
Versions []BuilderVersion `json:"versions"`
|
||||
}
|
||||
|
||||
// VersionsAsker asks builder_versions for a name's versions (a kept call, ADR 0668 §1, §2).
|
||||
type VersionsAsker func(ctx context.Context, name string) (BuilderVersionsAnswer, error)
|
||||
|
||||
// versionNameOf is the name the builder numbers an artifact under: its package's name in its ecosystem, else
|
||||
// <module>/<artifact> (images, bundles and archives).
|
||||
func versionNameOf(module string, a meshcatalog.Artifact) string {
|
||||
if a.Package != nil && a.Package.Name != "" {
|
||||
return a.Package.Name
|
||||
}
|
||||
return module + "/" + artifactName(a.Ref)
|
||||
}
|
||||
|
||||
// FillVersions fills the state versions once from builder_versions and says versions-filled (§372). A fill already made
|
||||
// is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding
|
||||
// call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
|
||||
func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return errors.New("the state versions is not kept here")
|
||||
}
|
||||
if !ix.versionsFilling.CompareAndSwap(false, true) {
|
||||
return nil // one fill runs at a time
|
||||
}
|
||||
defer ix.versionsFilling.Store(false)
|
||||
if _, _, done, err := ix.Heard.Get(ctx, VersionsFilledKey); err != nil || done {
|
||||
return err
|
||||
}
|
||||
if ix.Revisions == nil {
|
||||
return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts")
|
||||
}
|
||||
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first")
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type found1 struct {
|
||||
v BuilderVersion
|
||||
name string
|
||||
ref string
|
||||
}
|
||||
best := map[string][]found1{} // by module: each artifact's highest published version
|
||||
var unanswered error
|
||||
for _, e := range entries {
|
||||
var top string // the module's highest published version
|
||||
per := []found1{}
|
||||
for _, a := range e.Artifacts {
|
||||
name := versionNameOf(e.Module, a)
|
||||
ans, err := ask(ctx, name)
|
||||
if err != nil {
|
||||
unanswered = err
|
||||
continue
|
||||
}
|
||||
for _, v := range ans.Versions {
|
||||
if _, ok := versionParts(v.Version); !ok || v.State != "published" {
|
||||
continue
|
||||
}
|
||||
per = append(per, found1{v: v, name: name, ref: a.Ref})
|
||||
if top == "" || compareVersions(v.Version, top) > 0 {
|
||||
top = v.Version
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, f := range per {
|
||||
if f.v.Version == top {
|
||||
best[e.Module] = append(best[e.Module], f)
|
||||
}
|
||||
}
|
||||
}
|
||||
if unanswered != nil {
|
||||
ix.fillUnanswered(ctx, unanswered)
|
||||
return fmt.Errorf("builder_versions is not answered: %w", unanswered)
|
||||
}
|
||||
ix.answered(ctx, versionsFillSince)
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, "builder_versions answers"); err != nil {
|
||||
ix.logf("versions fill: the finding %s is not cleared: %v", BuilderVersionsCall, err)
|
||||
}
|
||||
}
|
||||
now := ix.now().UTC()
|
||||
filled := 0
|
||||
built := map[string]meshcatalog.VersionEntry{}
|
||||
claimed := map[string]time.Time{}
|
||||
trunk := map[string]meshcatalog.CatalogEntry{}
|
||||
for _, e := range entries {
|
||||
trunk[e.Module] = e
|
||||
}
|
||||
for module, fs := range best {
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: fs[0].v.Version, At: &now}
|
||||
for _, f := range fs {
|
||||
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: f.ref, Digest: f.v.Digest})
|
||||
if n, err := strconv.ParseInt(f.v.Revision, 10, 64); err == nil && n > v.Source {
|
||||
v.Source = n
|
||||
}
|
||||
if f.v.At.After(claimed[module]) {
|
||||
claimed[module] = f.v.At
|
||||
}
|
||||
}
|
||||
built[module] = v
|
||||
}
|
||||
// The edge entries (§377), each marked filled: where the stream running or the placement cannot be read the entries of kinds 2, 4
|
||||
// and 6 read "not yet delivered", as at a release, and the first delivery writes them.
|
||||
var running map[string]map[string]nodeengine.Running
|
||||
if ix.RunningWhole != nil {
|
||||
if running, err = ix.RunningWhole(ctx); err != nil {
|
||||
ix.logf("versions fill: the stream running is not read whole: %v", err)
|
||||
running = nil
|
||||
}
|
||||
}
|
||||
answers := map[string]BuilderVersionsAnswer{}
|
||||
for _, module := range slices.Sorted(maps.Keys(built)) {
|
||||
v := built[module]
|
||||
v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, running, answers)
|
||||
built[module] = v
|
||||
}
|
||||
for _, module := range slices.Sorted(maps.Keys(built)) {
|
||||
v := built[module]
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return fmt.Errorf("the version %s %s is not kept: %w", module, v.Version, err)
|
||||
}
|
||||
ix.keepFilledFacts(ctx, v, trunk[module], built, now)
|
||||
filled++
|
||||
}
|
||||
if ix.SayVersionsFilled != nil {
|
||||
ev := meshcatalog.VersionsFilledEvent{V: int64(keys.Version), At: now, Modules: int64(filled)}
|
||||
if err := ix.SayVersionsFilled(ctx, ev, meshcatalog.EventVersionsFilled); err != nil {
|
||||
return fmt.Errorf("versions-filled is not said: %w", err)
|
||||
}
|
||||
}
|
||||
if err := ix.Heard.Create(ctx, VersionsFilledKey, []byte(now.Format(time.RFC3339Nano))); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return err
|
||||
}
|
||||
ix.logf("versions: filled for %d modules from builder_versions, versions-filled said", filled)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillUnanswered writes call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
|
||||
func (ix *Indexer) fillUnanswered(ctx context.Context, cause error) {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, versionsFillSince, now)
|
||||
if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter {
|
||||
return
|
||||
}
|
||||
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", BuilderVersionsCall, since.UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The first fill of the catalog's versions asks %s of every artifact; it is not answered since %s (%v), so no version is released and the builder reserves nothing.", BuilderVersionsCall, since.UTC().Format(time.RFC3339), cause),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("versions fill: the finding %s is not written: %v", BuilderVersionsCall, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The edge entries of the first fill (design 47 G §377, A-78 c): each module's newest version gets its entries once, each marked
|
||||
// filled. Kinds 1 and 3 come from the dependency version published latest before the version's build was claimed, read with
|
||||
// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 from the version every machine placing the dependency
|
||||
// runs, present or away, the machines from the assigner's placement (A-62) and each node's last running from the node-engine seat's
|
||||
// stream read whole (§556); kind 5 as done for every acting part's running build.
|
||||
|
||||
// PlacementAsker asks the seat mesh-module-assigner's verb placement for a module (a read, A-62).
|
||||
type PlacementAsker func(ctx context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error)
|
||||
|
||||
// RunningReader reads the node-engine seat's stream running whole, every subject's last message (§556): by node, then module.
|
||||
type RunningReader func(ctx context.Context) (map[string]map[string]nodeengine.Running, error)
|
||||
|
||||
// filledEdges are the edge entries of the entry's newest version, which was filled as built; claimed is when its build was claimed.
|
||||
func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEntry, claimed time.Time, ask VersionsAsker, trunk map[string]meshcatalog.CatalogEntry,
|
||||
built map[string]meshcatalog.VersionEntry, running map[string]map[string]nodeengine.Running, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge {
|
||||
var out []meshcatalog.VersionEdge
|
||||
seen := func(e meshcatalog.VersionEdge) bool {
|
||||
return slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word })
|
||||
}
|
||||
for _, g := range entry.Edges {
|
||||
part := cmp.Or(g.Part, g.From)
|
||||
module := strings.SplitN(part, ":", 2)[0]
|
||||
if part == "" || module == entry.Module {
|
||||
continue
|
||||
}
|
||||
e := meshcatalog.VersionEdge{Kind: g.Kind, Part: part, Word: g.Word, Filled: true}
|
||||
switch g.Kind {
|
||||
case 1, 3:
|
||||
dep, ok := trunk[module]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var name string
|
||||
for _, a := range dep.Artifacts {
|
||||
if a.Ref == part {
|
||||
name = versionNameOf(module, a)
|
||||
}
|
||||
}
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
ans, ok := answers[name]
|
||||
if !ok {
|
||||
a, err := ask(ctx, name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ans, answers[name] = a, a
|
||||
}
|
||||
var best *BuilderVersion
|
||||
for i, v := range ans.Versions {
|
||||
if _, ok := versionParts(v.Version); !ok || v.State != "published" || v.At.After(claimed) {
|
||||
continue
|
||||
}
|
||||
if best == nil || v.At.After(best.At) {
|
||||
best = &ans.Versions[i]
|
||||
}
|
||||
}
|
||||
if best == nil {
|
||||
continue
|
||||
}
|
||||
e.Version, e.Digest, e.Stale = best.Version, best.Digest, true
|
||||
case 2, 4, 6:
|
||||
e.Version = notYetDelivered
|
||||
if v, ok := ix.runningVersion(ctx, module, trunk, built, running); ok {
|
||||
e.Version = v.Version
|
||||
for _, a := range v.Artifacts {
|
||||
if a.Artifact == part {
|
||||
e.Digest = a.Digest
|
||||
}
|
||||
}
|
||||
}
|
||||
case 5:
|
||||
e.Fact = "acted for the running build " + entry.Fp
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if !seen(e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runningVersion is the version of a module that every machine placing it runs, present or away, when each runs the newest
|
||||
// build; false where no machine places it, a placed machine runs none or another build, or the placement is not answered.
|
||||
func (ix *Indexer) runningVersion(ctx context.Context, module string, trunk map[string]meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry,
|
||||
running map[string]map[string]nodeengine.Running) (meshcatalog.VersionEntry, bool) {
|
||||
v, ok := built[module]
|
||||
e, known := trunk[module]
|
||||
if !ok || !known || e.Fp == "" || ix.Placement == nil {
|
||||
return v, false
|
||||
}
|
||||
placements, err := ix.Placement(ctx, module)
|
||||
if err != nil {
|
||||
ix.logf("versions fill: the placement of %s is not answered: %v", module, err)
|
||||
return v, false
|
||||
}
|
||||
nodes := 0
|
||||
for _, p := range placements {
|
||||
for _, n := range p.Nodes {
|
||||
nodes++
|
||||
r, ok := running[n][module]
|
||||
if !ok || r.Stopped || r.Fingerprint != e.Fp {
|
||||
return v, false
|
||||
}
|
||||
}
|
||||
}
|
||||
return v, nodes > 0
|
||||
}
|
||||
|
||||
// keepFilledFacts keeps what the first fill derived of a version as heard: for each entry of kinds 2, 4 and 6 read as delivered,
|
||||
// the dependency's version as its newest delivered one (A-68), and for each kind-5 edge the act for the running build (A-78 a).
|
||||
func (ix *Indexer) keepFilledFacts(ctx context.Context, v meshcatalog.VersionEntry, entry meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, at time.Time) {
|
||||
for _, e := range v.Edges {
|
||||
switch e.Kind {
|
||||
case 2, 4, 6:
|
||||
dep := built[strings.SplitN(e.Part, ":", 2)[0]]
|
||||
if e.Version == notYetDelivered || dep.Version != e.Version {
|
||||
continue
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for _, a := range dep.Artifacts {
|
||||
parts[a.Artifact] = a.Digest
|
||||
}
|
||||
d := meshwants.VersionDeliveredEvent{V: 1, Module: dep.Module, Version: dep.Version, Parts: parts, At: &at}
|
||||
if err := ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(dep.Module), d, func([]byte) bool { return false }); err != nil {
|
||||
ix.logf("versions fill: the delivered version of %s is not kept: %v", dep.Module, err)
|
||||
}
|
||||
case 5:
|
||||
data, _ := json.Marshal(Acted{V: 1, Part: e.Part, Build: entry.Fp, How: "filled", At: at})
|
||||
if err := write(ctx, ix.Heard, ActedKey(e.Part, v.Module), func(_ []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
return nil, false, nil
|
||||
}
|
||||
return data, false, nil
|
||||
}); err != nil {
|
||||
ix.logf("versions fill: the act for %s is not kept: %v", v.Module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// Design 47 G §349: a version is released once every artifact the module's entry names is published at it, said once as
|
||||
// version-released; a version lower than the highest released is recorded late and not said; each module keeps its newest
|
||||
// 100 versions, never its highest of any MAJOR.MINOR.
|
||||
func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
var said []meshcatalog.VersionReleasedEvent
|
||||
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now },
|
||||
SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error {
|
||||
said = append(said, e)
|
||||
return nil
|
||||
}}
|
||||
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Manifest: map[string]any{},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program"}, {Ref: "avahi:image"}}}}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := func(artifact, version string) {
|
||||
t.Helper()
|
||||
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "avahi", Artifact: artifact, Digest: "sha256:" + artifact + version, Version: version, Source: 7, Branch: "main"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pub("avahi:program", "0.1.5")
|
||||
if v, _ := ModuleVersions(ctx, ix.Versions, "avahi"); len(v) != 0 || len(said) != 0 {
|
||||
t.Fatalf("one of two artifacts published released %v, said %v", v, said)
|
||||
}
|
||||
pub("avahi:image", "0.1.5")
|
||||
pub("avahi:image", "0.1.5") // heard again
|
||||
v, _ := ModuleVersions(ctx, ix.Versions, "avahi")
|
||||
if len(v) != 1 || len(v[0].Artifacts) != 2 || v[0].Source != 7 || v[0].Late || len(said) != 1 || said[0].Version != "0.1.5" {
|
||||
t.Fatalf("released %+v, said %+v", v, said)
|
||||
}
|
||||
if keys, _ := ix.Heard.Keys(ctx); len(keys) != 0 {
|
||||
t.Errorf("the artifacts kept as published stay after the release: %v", keys)
|
||||
}
|
||||
// a late older version is recorded and not said
|
||||
pub("avahi:program", "0.1.2")
|
||||
pub("avahi:image", "0.1.2")
|
||||
v, _ = ModuleVersions(ctx, ix.Versions, "avahi")
|
||||
if len(v) != 2 || v[1].Version != "0.1.2" || !v[1].Late || len(said) != 1 {
|
||||
t.Fatalf("the late version: %+v, said %d", v, len(said))
|
||||
}
|
||||
// the newest 100 stay, and the highest of each MAJOR.MINOR
|
||||
for i := 6; i < 6+VersionsKept+5; i++ {
|
||||
pub("avahi:program", fmt.Sprintf("0.1.%d", i))
|
||||
pub("avahi:image", fmt.Sprintf("0.1.%d", i))
|
||||
}
|
||||
for _, mm := range []string{"0.2", "0.3"} {
|
||||
pub("avahi:program", mm+".0")
|
||||
pub("avahi:image", mm+".0")
|
||||
}
|
||||
v, _ = ModuleVersions(ctx, ix.Versions, "avahi")
|
||||
if len(v) != VersionsKept || v[0].Version != "0.3.0" {
|
||||
t.Fatalf("%d versions kept, newest %s", len(v), v[0].Version)
|
||||
}
|
||||
var oldest string
|
||||
for _, e := range v {
|
||||
oldest = e.Version
|
||||
}
|
||||
if compareVersions(oldest, "0.1.5") < 0 {
|
||||
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §372: the first fill records, once, each module's highest published version over its artifacts as released
|
||||
// and says versions-filled once; a builder_versions that does not answer holds it and, after five minutes, is the finding
|
||||
// call-unanswered; a fill made is not made again.
|
||||
func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
w := newFindings()
|
||||
var filledSaid []meshcatalog.VersionsFilledEvent
|
||||
ix := &Indexer{Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now },
|
||||
SayVersionsFilled: func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error {
|
||||
filledSaid = append(filledSaid, e)
|
||||
return nil
|
||||
}}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
|
||||
{Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}},
|
||||
{Ref: "mesh-sdk:ts", Package: &meshcatalog.ArtifactPackage{Ecosystem: "npm", Name: "@novox/mesh-sdk"}}}},
|
||||
"avahi": {Module: "avahi", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:image", Kind: "image"}}},
|
||||
"fresh": {Module: "fresh", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "fresh:image", Kind: "image"}}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
up := false
|
||||
ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) {
|
||||
if !up {
|
||||
return BuilderVersionsAnswer{}, errors.New("no responders")
|
||||
}
|
||||
pub := func(v, d string) BuilderVersion {
|
||||
return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9"}
|
||||
}
|
||||
switch name {
|
||||
case "git.novox.be/novox/mesh-sdk/go":
|
||||
return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.1.14", "sha256:go14"), {Version: "0.1.15", State: "refused"}}}, nil
|
||||
case "@novox/mesh-sdk":
|
||||
return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.1.18", "sha256:ts18")}}, nil
|
||||
case "avahi/image":
|
||||
return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.2.3", "sha256:av")}}, nil
|
||||
}
|
||||
return BuilderVersionsAnswer{Name: name}, nil
|
||||
}
|
||||
if err := ix.FillVersions(ctx, ask); err == nil || len(filledSaid) != 0 {
|
||||
t.Fatalf("a builder_versions unanswered filled: %v", err)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Fatal("a call unanswered for no time wrote its finding")
|
||||
}
|
||||
now = now.Add(5 * time.Minute)
|
||||
_ = ix.FillVersions(ctx, ask)
|
||||
if h := w.Held(); len(h) != 1 || h[0].ID != BuilderVersionsCall {
|
||||
t.Fatalf("the findings after five minutes: %+v", h)
|
||||
}
|
||||
up = true
|
||||
if err := ix.FillVersions(ctx, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Errorf("the finding stands after builder_versions answered: %+v", w.Held())
|
||||
}
|
||||
if len(filledSaid) != 1 || filledSaid[0].Modules != 2 {
|
||||
t.Fatalf("versions-filled said %+v", filledSaid)
|
||||
}
|
||||
sdk, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk")
|
||||
if len(sdk) != 1 || sdk[0].Version != "0.1.18" || len(sdk[0].Artifacts) != 1 || sdk[0].Artifacts[0].Artifact != "mesh-sdk:ts" || sdk[0].Source != 9 {
|
||||
t.Errorf("mesh-sdk's filled version: %+v (the higher of Go's 0.1.14 and npm's 0.1.18)", sdk)
|
||||
}
|
||||
if fresh, _ := ModuleVersions(ctx, ix.Versions, "fresh"); len(fresh) != 0 {
|
||||
t.Errorf("a module with none published is filled: %+v", fresh)
|
||||
}
|
||||
if err := ix.FillVersions(ctx, ask); err != nil || len(filledSaid) != 1 {
|
||||
t.Errorf("filled again: %v, said %d", err, len(filledSaid))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §374, §375, A-68: a release records, per edge, the part it was built against (kind 1, from package-published's
|
||||
// dependencies) or, once delivered, the delivered version it was ordered after (kinds 2 and 4); outdated answers an edge behind
|
||||
// by the part's digest alone, with the act rebuild only for a stale edge; a newer version that left the digest as it was
|
||||
// makes nothing behind.
|
||||
func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"lib": {Module: "lib", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}},
|
||||
"dep": {Module: "dep", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "dep:program"}}},
|
||||
"app": {Module: "app", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}},
|
||||
Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"},
|
||||
{Kind: 4, From: "dep:program", To: "app:program", Part: "dep:program", Word: "provision"}}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := func(module, version, digest string, deps ...meshbuilds.Dependency) {
|
||||
t.Helper()
|
||||
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: module, Artifact: module + ":" + map[string]string{"lib": "pkg", "dep": "program", "app": "program"}[module], Digest: digest, Version: version, Dependencies: deps}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
behind := func() []meshcatalog.Outdated {
|
||||
t.Helper()
|
||||
a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return a.Behind
|
||||
}
|
||||
pub("lib", "0.1.1", "d1")
|
||||
pub("dep", "0.1.1", "x1")
|
||||
pub("app", "0.1.1", "a1", meshbuilds.Dependency{Kind: 1, Part: "lib:pkg", Version: "0.1.1", Digest: "d1"})
|
||||
v, _ := ModuleVersions(ctx, ix.Versions, "app")
|
||||
if len(v[0].Edges) != 2 || v[0].Edges[0].Kind != 1 || !v[0].Edges[0].Stale || v[0].Edges[0].Word != "imports" || v[0].Edges[1].Version != "not yet delivered" {
|
||||
t.Fatalf("the release's edges: %+v", v[0].Edges)
|
||||
}
|
||||
pub("lib", "0.1.2", "d1") // a newer version, the part's digest as it was
|
||||
if b := behind(); len(b) != 0 {
|
||||
t.Fatalf("a newer version that left the digest as it was is behind: %+v", b)
|
||||
}
|
||||
pub("lib", "0.1.3", "d3")
|
||||
if b := behind(); len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Act != "rebuild" || b[0].Dependency.Artifact != "lib:pkg" || !strings.Contains(b[0].Newest, "d3") || !strings.Contains(b[0].Recorded, "d1") {
|
||||
t.Fatalf("behind after lib's new digest: %+v", b)
|
||||
}
|
||||
// delivery: dep 0.1.1 is delivered, then app 0.1.1 writes its kind 4 entry; dep 0.1.2 delivered later is behind, an ordering edge
|
||||
deliver := func(module, version string, parts map[string]string) {
|
||||
t.Helper()
|
||||
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: module, Version: version, Parts: parts}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
deliver("dep", "0.1.1", map[string]string{"dep:program": "x1"})
|
||||
deliver("app", "0.1.1", map[string]string{"app:program": "a1"})
|
||||
v, _ = ModuleVersions(ctx, ix.Versions, "app")
|
||||
if e := v[0].Edges[1]; e.Version != "0.1.1" || e.Digest != "x1" {
|
||||
t.Fatalf("the kind 4 entry after the first delivery: %+v", e)
|
||||
}
|
||||
deliver("dep", "0.1.2", map[string]string{"dep:program": "x2"})
|
||||
deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) // delivered again: written once, not changed
|
||||
v, _ = ModuleVersions(ctx, ix.Versions, "app")
|
||||
if e := v[0].Edges[1]; e.Version != "0.1.1" {
|
||||
t.Errorf("an entry written at the first delivery changed: %+v", e)
|
||||
}
|
||||
b := behind()
|
||||
if len(b) != 2 || b[1].Kind != 4 || b[1].Act != "" || b[1].Dependency.Artifact != "dep:program" {
|
||||
t.Fatalf("behind after dep's second delivery: %+v", b)
|
||||
}
|
||||
// the newest toolchain-agreed per language is kept as heard
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw, _, found, _ := ix.Heard.Get(ctx, ToolchainPrefix+"go"); !found || !strings.Contains(string(raw), "1.26") {
|
||||
t.Errorf("toolchain.go holds %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §354: an input is kept under its write time and sha256, read back by the verb input with its digest checked;
|
||||
// the keys older than a day among the 20 oldest are deleted at each new keep.
|
||||
func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }}
|
||||
d1, err := ix.KeepInput(ctx, "ask-1", []byte("first"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1})
|
||||
if err != nil || a.Digest != d1 || a.Input != "Zmlyc3Q=" {
|
||||
t.Fatalf("the input read back: %+v, %v", a, err)
|
||||
}
|
||||
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: "sha256:none"}); err == nil {
|
||||
t.Error("an input never kept is answered")
|
||||
}
|
||||
now = now.Add(23 * time.Hour)
|
||||
if _, err := ix.KeepInput(ctx, "ask-2", []byte("second")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputsOnly := func() []string {
|
||||
var out []string
|
||||
all, _ := ix.Inputs.Keys(ctx)
|
||||
for _, k := range all {
|
||||
if !strings.HasPrefix(k, InputsAskPrefix) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
if keys := inputsOnly(); len(keys) != 2 {
|
||||
t.Fatalf("inside a day: %v", keys)
|
||||
}
|
||||
now = now.Add(2 * time.Hour)
|
||||
d3, _ := ix.KeepInput(ctx, "ask-3", []byte("third"))
|
||||
if keys := inputsOnly(); len(keys) != 2 {
|
||||
t.Fatalf("after a day the first stays: %v", keys)
|
||||
}
|
||||
if all, _ := ix.Inputs.Keys(ctx); slices.Contains(all, InputsAskPrefix+"ask-1") {
|
||||
t.Errorf("the name by ask of the swept input stays: %v", all)
|
||||
}
|
||||
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil {
|
||||
t.Error("the input a day old is still answered")
|
||||
}
|
||||
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d3}); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-80, §73: what the indexer heard lives in the state heard, each key part escaped as a name, so a dotted module
|
||||
// name parses and nothing of it is written to the own-build state.
|
||||
func TestWhatIsHeardIsKeptEscapedInTheStateHeard(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
|
||||
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "a.b", Version: "1.2.3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := ix.deliveredOf(ctx, "a.b"); !ok {
|
||||
t.Error("the delivered version of a dotted module is not read back")
|
||||
}
|
||||
got, _ := ix.Heard.Keys(ctx)
|
||||
slices.Sort(got)
|
||||
if want := []string{"delivered.a=2eb", "toolchain.go"}; !slices.Equal(got, want) {
|
||||
t.Errorf("the keys of heard are %v, want %v", got, want)
|
||||
}
|
||||
if own, _ := ix.OwnBuildState.Keys(ctx); len(own) != 0 {
|
||||
t.Errorf("the own-build state holds %v", own)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §375, A-78, A-80: outdated says a kind-3 edge behind when the agreed toolchain of its artifact's language names another
|
||||
// version, a kind-5 edge when the acting part has not acted for the dependent's current build (a wait-ended that ended ran or
|
||||
// released), and a kind-6 edge when the delivered version's entry no longer serves the word.
|
||||
func TestOutdatedAnswersKinds3And5And6(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
|
||||
keep := func(n, source int64, serves []string, fp string) {
|
||||
t.Helper()
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"prov": {Module: "prov", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "prov:program"}}, Serves: serves},
|
||||
"actor": {Module: "actor", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}},
|
||||
"app": {Module: "app", Manifest: map[string]any{}, Fp: fp, Artifacts: []meshcatalog.Artifact{{Ref: "app:program", Language: "go"}},
|
||||
Edges: []meshcatalog.Edge{{Kind: 3, From: "tc:program", To: "app:program", Part: "tc:program"},
|
||||
{Kind: 5, From: "actor:program", To: "app:program", Part: "actor:program"},
|
||||
{Kind: 6, From: "prov:program", To: "app:program", Part: "prov:program", Word: "seat:s.verb"}}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: n, Source: source, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
keep(1, 10, []string{"s.verb"}, "fp1")
|
||||
put := func(module string, v meshcatalog.VersionEntry) {
|
||||
t.Helper()
|
||||
v.Module, v.V = module, 1
|
||||
data, _ := json.Marshal(v)
|
||||
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put("prov", meshcatalog.VersionEntry{Version: "0.1.1", Source: 10})
|
||||
put("app", meshcatalog.VersionEntry{Version: "0.1.1", Edges: []meshcatalog.VersionEdge{
|
||||
{Kind: 3, Part: "tc:program", Version: "1.25", Digest: "t1", Stale: true},
|
||||
{Kind: 5, Part: "actor:program"},
|
||||
{Kind: 6, Part: "prov:program", Word: "seat:s.verb", Version: "0.1.1"}}})
|
||||
behind := func() map[int64]meshcatalog.Outdated {
|
||||
t.Helper()
|
||||
a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := map[int64]meshcatalog.Outdated{}
|
||||
for _, o := range a.Behind {
|
||||
m[o.Kind] = o
|
||||
}
|
||||
return m
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := behind()
|
||||
if _, ok := b[3]; ok {
|
||||
t.Errorf("kind 3 behind at the agreed version: %+v", b[3])
|
||||
}
|
||||
if o, ok := b[5]; !ok || o.Act != "" {
|
||||
t.Errorf("kind 5 is not behind before the acting part acted: %+v", b)
|
||||
}
|
||||
wait := func(how, step, build string, since time.Time) {
|
||||
t.Helper()
|
||||
if err := ix.HearWaitEnded(ctx, meshwants.WaitEndedEvent{Part: "actor:program", Module: "app", Step: step, How: how, Version: build, Since: since}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
at := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
wait("superseded", "compose", "fp1", at)
|
||||
wait("ran", "delivery", "fp1", at)
|
||||
if _, ok := behind()[5]; !ok {
|
||||
t.Error("a superseded wait or a delivery wait counted as an act")
|
||||
}
|
||||
wait("ran", "compose", "fp1", at)
|
||||
if _, ok := behind()[5]; ok {
|
||||
t.Error("kind 5 is behind after the act for the current build")
|
||||
}
|
||||
keep(2, 20, []string{"s.other"}, "fp2")
|
||||
if o, ok := behind()[5]; !ok || !strings.Contains(o.Newest, "fp2") {
|
||||
t.Errorf("kind 5 after the dependent's new build: %+v", o)
|
||||
}
|
||||
wait("released", "compose", "fp2", at.Add(time.Minute))
|
||||
if _, ok := behind()[5]; ok {
|
||||
t.Error("a released wait is not an act")
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o, ok := behind()[3]; !ok || o.Act != "rebuild" || o.Newest != "1.26" || !strings.Contains(o.Recorded, "1.25") {
|
||||
t.Errorf("kind 3 after the agreed toolchain changed: %+v", o)
|
||||
}
|
||||
// kind 6: prov 0.1.2 built at source 20 serves another word only
|
||||
if _, ok := behind()[6]; ok {
|
||||
t.Error("kind 6 behind with no newer delivered version")
|
||||
}
|
||||
put("prov", meshcatalog.VersionEntry{Version: "0.1.2", Source: 20})
|
||||
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "prov", Version: "0.1.2", Parts: map[string]string{"prov:program": "p2"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o, ok := behind()[6]; !ok || o.Act != "" || !strings.Contains(o.Newest, "no longer serves") {
|
||||
t.Errorf("kind 6 after a version that serves no more: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §377, A-78 c: the first fill gives each module's newest version its edge entries, each marked filled: kind 1 from the
|
||||
// dependency version published latest before the build was claimed, kinds 2, 4 and 6 from the version every placing machine runs,
|
||||
// read from the assigner's placement and the stream running, kind 5 as acted for the running build; so the first outdated says only what is truly behind.
|
||||
func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t0 := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
placed := map[string][]string{"prov": {"n1", "n2"}}
|
||||
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 },
|
||||
Placement: func(_ context.Context, m string) ([]meshmoduleassigner.ModulePlacement, error) {
|
||||
return []meshmoduleassigner.ModulePlacement{{Branch: "trunk", Nodes: placed[m]}}, nil
|
||||
},
|
||||
RunningWhole: func(context.Context) (map[string]map[string]nodeengine.Running, error) {
|
||||
return map[string]map[string]nodeengine.Running{"n1": {"prov": {Fingerprint: "fp-prov"}}, "n2": {"prov": {Fingerprint: "fp-prov"}}}, nil
|
||||
}}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"lib": {Module: "lib", Manifest: map[string]any{}, Fp: "fp-lib", Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x/lib"}}}},
|
||||
"actor": {Module: "actor", Manifest: map[string]any{}, Fp: "fp-actor", Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}},
|
||||
"prov": {Module: "prov", Manifest: map[string]any{}, Fp: "fp-prov", Artifacts: []meshcatalog.Artifact{{Ref: "prov:program"}}},
|
||||
"app": {Module: "app", Manifest: map[string]any{}, Fp: "fp-app", Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}},
|
||||
Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true},
|
||||
{Kind: 4, From: "prov:program", To: "app:program", Part: "prov:program"},
|
||||
{Kind: 5, From: "actor:program", To: "app:program", Part: "actor:program"}}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) {
|
||||
pub := func(v, d string, at time.Time) BuilderVersion {
|
||||
return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9", At: at}
|
||||
}
|
||||
switch name {
|
||||
case "x/lib":
|
||||
return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.1.1", "sha256:l1", t0), pub("0.1.2", "sha256:l2", t0.Add(2*time.Hour))}}, nil
|
||||
case "app/program":
|
||||
return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.3.1", "sha256:a1", t0.Add(time.Hour))}}, nil
|
||||
}
|
||||
return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.9.1", "sha256:"+name, t0)}}, nil
|
||||
}
|
||||
if err := ix.FillVersions(ctx, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app, _ := ModuleVersions(ctx, ix.Versions, "app")
|
||||
if len(app) != 1 || len(app[0].Edges) != 3 {
|
||||
t.Fatalf("app's filled version: %+v", app)
|
||||
}
|
||||
for _, e := range app[0].Edges {
|
||||
if !e.Filled {
|
||||
t.Errorf("a filled entry is not marked: %+v", e)
|
||||
}
|
||||
switch e.Kind {
|
||||
case 1:
|
||||
if e.Version != "0.1.1" || e.Digest != "sha256:l1" || !e.Stale {
|
||||
t.Errorf("kind 1 is the version published before the build was claimed: %+v", e)
|
||||
}
|
||||
case 4:
|
||||
if e.Version != "0.9.1" || e.Digest != "sha256:prov/program" {
|
||||
t.Errorf("kind 4 is the version every machine runs: %+v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
// lib's 0.1.2 was published after app's build was claimed: that is behind, and nothing else (the delivered kind 4 and the act of kind 5 are not)
|
||||
if a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{}); err != nil || len(a.Behind) != 1 || a.Behind[0].Kind != 1 {
|
||||
t.Errorf("outdated after the first fill: %+v, %v", a.Behind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §354, A-78 d: an input is also keyed by its run ask's id, so the build seat's step-ended of that ask deletes it, and
|
||||
// the key by ask no other read counts as an input.
|
||||
func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }}
|
||||
d1, err := ix.KeepInput(ctx, "a.1", []byte("first"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d2, _ := ix.KeepInput(ctx, "a.2", []byte("second"))
|
||||
if _, err := ix.KeepInput(ctx, "", []byte("x")); err == nil {
|
||||
t.Error("an input kept for no ask")
|
||||
}
|
||||
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "a.1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "a.1"}); err != nil {
|
||||
t.Errorf("an end heard again: %v", err)
|
||||
}
|
||||
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil {
|
||||
t.Error("the input of an ended ask is still answered")
|
||||
}
|
||||
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d2}); err != nil {
|
||||
t.Errorf("the input of another ask: %v", err)
|
||||
}
|
||||
if got, _ := ix.Inputs.Keys(ctx); len(got) != 2 {
|
||||
t.Errorf("keys after one end: %v", got)
|
||||
}
|
||||
if a, err := InputsReadTool(ix.Inputs).Run(map[string]any{}); err != nil || len(a.(InputsAnswer).Inputs) != 1 {
|
||||
t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
@@ -130,7 +130,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
{Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
|
||||
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
|
||||
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
|
||||
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
|
||||
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
|
||||
@@ -164,9 +165,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments",
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments",
|
||||
"mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery.source-moved", "mesh-wants.wanted", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
"mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -178,7 +179,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
|
||||
@@ -201,8 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool) // design 47 G §667, §8
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || slices.Contains(m.Tools, "modules") {
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool) // design 47 G §667, §8
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 47 || slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
@@ -306,7 +307,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools
|
||||
// bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch"}) {
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions", "outdated", "input"}) {
|
||||
t.Errorf("claims = %+v, want mesh-catalog with its seven verbs (entry: design 47 G §675)", m.Claims)
|
||||
}
|
||||
for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} {
|
||||
|
||||
@@ -19,7 +19,10 @@
|
||||
"register",
|
||||
"policy",
|
||||
"watch",
|
||||
"unwatch"
|
||||
"unwatch",
|
||||
"versions",
|
||||
"outdated",
|
||||
"input"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -69,7 +72,8 @@
|
||||
"indexer_named_drop",
|
||||
"indexer_proposals",
|
||||
"indexer_proposal_drop",
|
||||
"indexer_seed_again"
|
||||
"indexer_seed_again",
|
||||
"indexer_inputs"
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
@@ -80,6 +84,11 @@
|
||||
"mesh-delivery-checker.preview.written",
|
||||
"mesh-delivery-checker.try.ended",
|
||||
"mesh-delivery-checker.try.started",
|
||||
"mesh-builds.package-published",
|
||||
"mesh-builds.toolchain-agreed",
|
||||
"mesh-wants.version-delivered",
|
||||
"node-build-go.step-ended",
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-delivery-checker.verdict",
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery.source-moved",
|
||||
@@ -111,7 +120,10 @@
|
||||
{ "name": "entry-index", "history": 1, "max-mib": 8 },
|
||||
{ "name": "named", "history": 1, "max-mib": 4 },
|
||||
{ "name": "registrations", "history": 1 },
|
||||
{ "name": "proposals", "history": 1, "max-mib": 4 }
|
||||
{ "name": "proposals", "history": 1, "max-mib": 4 },
|
||||
{ "name": "versions", "history": 1, "max-mib": 64 },
|
||||
{ "name": "build-edges", "history": 1 },
|
||||
{ "name": "inputs", "history": 1 }
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 },
|
||||
@@ -155,7 +167,11 @@
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-module-assigner.placement",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-watchdogs.presence"
|
||||
"seat:mesh-watchdogs.presence",
|
||||
"seat:node-build-go.steps",
|
||||
"seat:node-build-node.steps",
|
||||
"seat:node-build-image.steps",
|
||||
"mesh-delivery-builder.builder_versions"
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
|
||||
Reference in New Issue
Block a user