Merge branch 'feat/47-G-indexer6b' into feat/47-G-join8

# Conflicts:
#	modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go
#	modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
#	modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
#	modules/mesh-delivery-indexer/internal/index/pending.go
#	modules/mesh-delivery-indexer/internal/index/pending_test.go
#	modules/mesh-delivery-indexer/manifest_test.go
This commit is contained in:
2026-11-12 00:12:54 +01:00
33 changed files with 3309 additions and 73 deletions
@@ -288,7 +288,9 @@ func main() {
tools = append(tools, catalogTools(trunk.State)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}))
tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
index.InputTool(keptRead{stdio.State(index.InputsState)}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}))
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"})
return err
@@ -1474,3 +1474,74 @@ func TestIndexerKeepsWhatAWantAndARunningBuildNameAndLetsAGoneOneLeave(t *testin
t.Errorf("named holds %v after the want of n2 was said removed, want n1.avahi alone", got)
}
}
// TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision: the seat's verb source lists a second watched branch,
// beta, with the merge of mesh-catalog it names as its head (design 47 G §13, §14, §291, A-93); the indexer started with the
// branch listed reads source.beta, takes the head's merged message in as a pure computation, and keeps branch.beta.1, which
// the branch follower (index.BranchView, as the branch's builds read it) reads; the trunk's modules are left.
func TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const merge = "89abcdef0123456789abcdef0123456789abcdef"
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "beta", Number: 301,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0",
Trees: map[string]string{"modules/avahi": "betatree"}, Manifests: map[string]string{"modules/avahi": string(manifest)},
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}})
if err != nil {
t.Fatal(err)
}
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
}, func(ctx context.Context, js jetstream.JetStream) {
seedBuildModules(t, ctx, js, nil)
source := func(m *nats.Msg) {
var ask struct{ Branch string }
_ = json.Unmarshal(m.Data, &ask)
answer := meshdelivery.SourceAnswer{Branch: ask.Branch}
switch ask.Branch {
case "":
answer.Branches = []string{"main", "beta"}
case "beta":
answer.Group, answer.Revision = "g", 9
answer.Repositories = []meshdelivery.SourceHead{{Repository: "novox/mesh-catalog", Head: merge, Trunk: "beta"}}
}
raw, _ := json.Marshal(map[string]any{"result": answer, "node": "n1"})
_ = m.Respond(raw)
}
bridge := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data)}, "node": "n1"})
_ = m.Respond(raw)
}
for subject, h := range map[string]nats.MsgHandler{
"mesh.seat.mesh-delivery.call.source.>": source, "mesh.seat.mesh-delivery.tool.source": source,
"mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>": bridge, "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message": bridge,
} {
if _, err := js.Conn().Subscribe(subject, h); err != nil {
t.Fatal(err)
}
}
})
objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects)
var view index.Store
var at int64
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && at == 0; time.Sleep(50 * time.Millisecond) {
if v, _, n, err := index.BranchView(ctx, objects, "beta"); err == nil {
view, at = v, n
}
}
if at != 1 {
t.Fatalf("the branch follower reads beta at revision %d within 20 s of the start, want 1", at)
}
value, _, found, err := view.Get(ctx, "avahi")
if err != nil || !found {
t.Fatalf("branch.beta.1 holds no avahi (%v)", err)
}
if m, err := meshcatalogclient.DecodeModule(value); err != nil || m.Commit != merge || m.Inputs2 == nil {
t.Errorf("avahi at the branch's head = %+v (%v), want commit %s with its inputs", m, err, merge)
}
if e, err := modules.Get(ctx, "avahi"); err == nil {
t.Errorf("modules.avahi = %s, want the trunk's modules left", e.Value())
}
}
@@ -22,7 +22,10 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -397,6 +400,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// appears; until then indexer_requirements answers none, and the builder's floor waits on its answer.
requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf)
// The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes.
// The state heard (A-80): the forge messages handled (key forge) and what the indexer heard and compares later (the time a call is
// unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version).
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "what the indexer heard is not kept", logger.Printf)}
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
@@ -430,6 +436,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
})
readWhenBound(seatReader, seatDefs, marks)
var fillFollow atomic.Pointer[versionsFill]
// Kept while named (design 47 G §675): the want and the running build that name a fingerprint, heard below.
named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.NamedBucket, "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf)
ix := &index.Indexer{NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
@@ -437,12 +444,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
// Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go).
SayCatalogRevision: refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow),
SayCatalogRevision: afterRevision(refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), &fillFollow),
SayFill: sayFill(conn, cred),
// The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released.
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)},
BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.BuildEdgesBucket, "no build of another watched branch keeps its edges", logger.Printf)},
Inputs: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)},
SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred),
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
@@ -453,10 +465,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.RegistrationsBucket, "no registration is kept or written", logger.Printf)},
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)},
routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf)
registry.Pruned = ix.PruneBranchBuilds
ix.Registry = registry
named.Changed = ix.RemoveUnlisted
ix.SeedModules = func(context.Context) ([]index.SeedModule, error) {
return nil, errors.New("the first registrations are not composed by this build: what the build seats answer steps with is not read yet (design question)")
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKept), logf: logger.Printf}
fillFollow.Store(fill)
fill.kick()
ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) {
r := seatReader.Reader()
if r == nil {
return nil, errors.New("the seat list is not read yet: the seed waits for it")
}
return ix.SeedModulesFrom(ctx, r.Definitions(), stepsVia(calls.CallKept))
}
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
@@ -493,7 +514,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
// meanwhile is read after.
source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, logf: logger.Printf}
source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf}
go source.catchUp(ctx)
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// assignments is settled first, so the toolchains settle through the gate already knowing it.
@@ -666,6 +687,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
_ = m.Ack() // nothing else the indexer consumes rides the shared stream
}
}
if err := hub.Handle(meshbuilds.SubjectPackagePublished, func(m jetstream.Msg) { heardPackagePublished(ctx, ix, m, logger.Printf) }); err != nil {
return err
}
if err := hub.Handle(meshbuilds.SubjectToolchainAgreed, func(m jetstream.Msg) { heardToolchainAgreed(ctx, ix, m, logger.Printf) }); err != nil {
return err
}
if err := hub.Handle(meshwants.SubjectVersionDelivered, func(m jetstream.Msg) { heardVersionDelivered(ctx, ix, m, logger.Printf) }); err != nil {
return err
}
if err := hub.Handle(meshwants.SubjectWaitEnded, func(m jetstream.Msg) { heardWaitEnded(ctx, ix, m, logger.Printf) }); err != nil {
return err
}
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(ctx, ix, m, logger.Printf) }); err != nil {
return err
}
if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil {
return err
}
@@ -726,7 +762,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
// What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds
// (issue 1790): until then no message is written there, and the rest of the indexer runs.
heard := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
heard := heardStore
// Each change of the states its readers read is said on its views family (ADR 0615 §5), from the mark each state was
// said up to; a first view (the tools bundle) answers at that mark.
startViews(ctx, js, cred, marks, writer,
@@ -191,6 +191,9 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(
s = &sourceFollow{
read: func(branch string) (meshdelivery.SourceAnswer, error) {
order = append(order, "read "+branch)
if branch == "" { // given no branch the seat lists the branches it keeps a source of (A-93)
return meshdelivery.SourceAnswer{Branches: []string{"main", "beta", "feat=2e1"}}, nil
}
if revision == 0 {
return meshdelivery.SourceAnswer{}, fmt.Errorf("source.main is not readable")
}
@@ -205,15 +208,21 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(
order = append(order, fmt.Sprintf("apply %s %d", a.Branch, a.Revision))
return nil
},
applyBranch: func(_ context.Context, branch string, a meshdelivery.SourceAnswer) error {
order = append(order, fmt.Sprintf("branch %s %d", branch, a.Revision))
return nil
},
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) },
}
s.catchUp(context.Background())
want := []string{"read main", "apply main 4", "read main", "apply main 4"} // once, after the catch-up; beta is left
// The trunk, then each other watched branch the seat lists (a name unescaped), then what was heard meanwhile once each,
// beta among them: main through the take-in, every other branch into its own revision (A-93).
want := []string{"read main", "apply main 4", "read ", "read beta", "branch beta 4", "read feat=2e1", "branch feat.1 4", "read main", "apply main 4", "read beta", "branch beta 4"}
if !reflect.DeepEqual(order, want) {
t.Errorf("order = %v, want %v", order, want)
}
s.moved(context.Background(), "main") // after the catch-up: read at once
if len(order) != 6 {
if len(order) != 13 {
t.Errorf("a source-moved after the catch-up was not read at once: %v", order)
}
order, revision = nil, 0
@@ -237,7 +237,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
}
// trunkBranch is the branch pull requests are open into.
const trunkBranch = "main"
const trunkBranch = index.TrunkBranch
// openVia asks the forge which pull requests into the trunk are open now (the seat git's open-pulls), every repository's.
// An answer that lacks the list is refused, never read as none.
@@ -8,6 +8,7 @@ import (
"sync"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"github.com/nats-io/nats.go/jetstream"
)
@@ -40,17 +41,31 @@ func sourceVia(call toolCaller) func(branch string) (meshdelivery.SourceAnswer,
type sourceFollow struct {
read func(branch string) (meshdelivery.SourceAnswer, error)
apply func(context.Context, meshdelivery.SourceAnswer) error
logf func(string, ...any)
// applyBranch takes a watched branch other than the trunk's in (design 47 G §291, A-93); branch is its name, unescaped.
applyBranch func(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error
logf func(string, ...any)
mu sync.Mutex
caught bool
waiting map[string]bool
}
// catchUp reads the trunk branch's source revision and applies it, then the branches a source-moved named meanwhile. A
// read that fails is said; the catch-up ends all the same, and the next source-moved reads again.
// catchUp reads the trunk branch's source revision and applies it, then every other branch the seat keeps a source of (the
// answer given no branch lists them, A-93), then the branches a source-moved named meanwhile. A read that fails is said;
// the catch-up ends all the same, and the next source-moved reads again.
func (s *sourceFollow) catchUp(ctx context.Context) {
s.take(ctx, trunkBranch)
var others []string
if list, err := s.read(""); err != nil {
s.logf("the branches the source names are not read, and each is read at its next source-moved: %v", err)
} else {
others = list.Branches
}
for _, branch := range others {
if branch != trunkBranch {
s.take(ctx, branch)
}
}
s.mu.Lock()
s.caught = true
branches := s.waiting
@@ -76,17 +91,28 @@ func (s *sourceFollow) moved(ctx context.Context, branch string) {
s.take(ctx, branch)
}
// take reads one branch's source revision and applies it. A branch other than the trunk's is left until the indexer keeps
// a catalog graph per watched branch.
// take reads one branch's source revision (branch as the key name source.<branch> has it) and applies it: the trunk's
// through the take-in, another watched branch's into its own revision (A-93).
func (s *sourceFollow) take(ctx context.Context, branch string) {
if branch != trunkBranch {
return
}
a, err := s.read(branch)
if err != nil {
s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err)
return
}
if branch != trunkBranch {
name, err := keys.UnescapeName(branch)
if err != nil {
s.logf("source.%s names no branch: %v", branch, err)
return
}
if s.applyBranch == nil {
return
}
if err := s.applyBranch(ctx, name, a); err != nil {
s.logf("source.%s at revision %d could not be taken into the branch's revision: %v", branch, a.Revision, err)
}
return
}
if err := s.apply(ctx, a); err != nil {
s.logf("source.%s at revision %d could not be taken into the modules: %v", branch, a.Revision, err)
}
@@ -0,0 +1,28 @@
package main
import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"mesh-delivery-indexer/internal/index"
)
// stepsVia asks a build seat's verb steps through call (process.Conn.CallKept, ADR 0668 §1) and reads its answer (design
// 47 G §704); the seats' verbs have one form, so nodebuildgo's contract reads each seat's answer.
func stepsVia(call toolCaller) index.StepsCaller {
return func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) {
verb := "seat:" + seat + "." + index.StepsVerb
raw, err := call(verb, in)
if err != nil {
return nodebuildgo.StepsAnswer{}, err
}
var a nodebuildgo.StepsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return a, fmt.Errorf("the answer of %s is not its steps: %w", verb, err)
}
return a, nil
}
}
@@ -0,0 +1,235 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
)
// saySeatEvent publishes body as the seat mesh-catalog's event under the bus's duplicate id, as every event of the seat is
// said: published, never asked, and a refusal said on the connection's error handler.
func saySeatEvent(conn *nats.Conn, cred process.Account, event, id string, body any) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(seatEventSubject(event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data = data
return conn.PublishMsg(msg)
}
// sayVersionReleased says version-released (design 47 G §349) on mesh.seat.mesh-catalog.event.version-released.
func sayVersionReleased(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionReleasedEvent, string) error {
return func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error {
return saySeatEvent(conn, cred, meshcatalog.EventVersionReleased, id, e)
}
}
// heardPackagePublished takes one package-published of the seat mesh-builds into the catalog's versions (design 47 G §349):
// one that does not decode is said and acknowledged; one that cannot be kept is left, so the bus gives it again.
func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
var p meshbuilds.PackagePublished
if err := json.Unmarshal(m.Data(), &p); err != nil {
logf("%s holds what is no package-published, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := ix.HearPackagePublished(ctx, p); err != nil {
logf("versions: the package-published of %s %s could not be kept, so the bus gives it again: %v", p.Artifact, p.Version, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// heardToolchainAgreed keeps the newest toolchain-agreed of its language (A-68); one that does not decode is said and
// acknowledged, one that cannot be kept is left, so the bus gives it again.
func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
var t meshbuilds.ToolchainAgreed
if err := json.Unmarshal(m.Data(), &t); err != nil {
logf("%s holds what is no toolchain-agreed, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := ix.HearToolchainAgreed(ctx, t); err != nil {
logf("versions: the toolchain-agreed of %s could not be kept, so the bus gives it again: %v", t.Language, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// heardVersionDelivered writes the delivered release's entries of kinds 2, 4 and 6 and keeps the newest delivered version of
// its module (design 47 G §374, A-68), as heardToolchainAgreed keeps its event.
func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
var d meshwants.VersionDeliveredEvent
if err := json.Unmarshal(m.Data(), &d); err != nil {
logf("%s holds what is no version-delivered, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := ix.HearVersionDelivered(ctx, d); err != nil {
logf("versions: the version-delivered of %s %s could not be kept, so the bus gives it again: %v", d.Module, d.Version, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// heardWaitEnded keeps the act of an acting part's ended wait for kind 5 (A-78 a), as heardToolchainAgreed keeps its event.
func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
var w meshwants.WaitEndedEvent
if err := json.Unmarshal(m.Data(), &w); err != nil {
logf("%s holds what is no wait-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := ix.HearWaitEnded(ctx, w); err != nil {
logf("versions: the wait-ended of %s for %s could not be kept, so the bus gives it again: %v", w.Part, w.Module, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// heardStepEnded deletes the input kept for the run ask a build seat's step-ended names (design 47 G §354, A-78 d).
func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
var e nodebuildgo.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := ix.HearInputEnded(ctx, e); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// sayVersionsFilled says versions-filled (design 47 G §372) once, on the seat's stream that keeps it with no age limit.
func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionsFilledEvent, string) error {
return func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error {
return saySeatEvent(conn, cred, meshcatalog.EventVersionsFilled, id, e)
}
}
// versionsVia asks the builder's tool builder_versions for one name as a kept call (ADR 0668 §1, §2) and reads its answer.
func versionsVia(call toolCaller) index.VersionsAsker {
return func(_ context.Context, name string) (index.BuilderVersionsAnswer, error) {
var a index.BuilderVersionsAnswer
raw, err := call(index.BuilderVersionsCall, map[string]any{"name": name})
if err != nil {
return a, err
}
if err := json.Unmarshal(raw, &a); err != nil {
return a, fmt.Errorf("the answer of %s is not a name's versions: %w", index.BuilderVersionsCall, err)
}
return a, nil
}
}
// versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each
// catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next.
type versionsFill struct {
ix *index.Indexer
ctx context.Context
ask index.VersionsAsker
logf func(string, ...any)
}
func (f *versionsFill) kick() {
go func() {
if err := f.ix.FillVersions(f.ctx, f.ask); err != nil {
f.logf("versions: the first fill waits: %v", err)
}
}()
}
// afterRevision runs the fill's kick after each catalog revision kept, once the fill is set.
func afterRevision(say func(context.Context, meshcatalog.CatalogRevisionEvent) error, fill *atomic.Pointer[versionsFill]) func(context.Context, meshcatalog.CatalogRevisionEvent) error {
return func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error {
err := say(ctx, e)
if f := fill.Load(); f != nil {
f.kick()
}
return err
}
}
// placementVia asks the seat mesh-module-assigner's verb placement for one module (A-62), a read.
func placementVia(call toolCaller) index.PlacementAsker {
return func(_ context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) {
raw, err := call(meshmoduleassigner.Seat+"."+meshmoduleassigner.VerbPlacement, map[string]any{"module": module})
if err != nil {
return nil, fmt.Errorf("%s.%s: %w", meshmoduleassigner.Seat, meshmoduleassigner.VerbPlacement, err)
}
var a meshmoduleassigner.PlacementAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of placement is not a module's placements: %w", err)
}
return a.Placements, nil
}
}
// runningWholeVia reads every subject's last message of the node-engine seat's stream running through an ordered consumer made for
// that one read (design 47 G §556), by node and module.
func runningWholeVia(js jetstream.JetStream) index.RunningReader {
return func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) {
stream, err := js.Stream(ctx, nodeengine.StreamRunning)
if err != nil {
return nil, fmt.Errorf("the stream %s cannot be read whole: %w", nodeengine.StreamRunning, err)
}
cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
if err != nil {
return nil, fmt.Errorf("the ordered consumer for the whole read of %s cannot be made: %w", nodeengine.StreamRunning, err)
}
out := map[string]map[string]nodeengine.Running{}
for {
batch, err := cons.FetchNoWait(256)
if err != nil {
return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err)
}
n := 0
for m := range batch.Messages() {
n++
var r nodeengine.Running
if json.Unmarshal(m.Data(), &r) != nil || r.Node == "" || r.Module == "" {
continue
}
if out[r.Node] == nil {
out[r.Node] = map[string]nodeengine.Running{}
}
out[r.Node][r.Module] = r
}
if err := batch.Error(); err != nil {
return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err)
}
if n == 0 {
return out, nil
}
}
}
}
+1 -1
View File
@@ -70,7 +70,7 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
@@ -0,0 +1,243 @@
package index
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The builds of a watched branch other than the first release no version (design 47 G §371); the indexer keeps each one's
// edge entries in the state build-edges, under <module>.<branch>, and what the branch's newest build of the module
// published as heard, branch-build.<module>.<branch> in the state heard (§374, A-83).
const (
BuildEdgesState = "build-edges"
BuildEdgesBucket = Module + "_" + BuildEdgesState
// BranchBuildPrefix opens branch-build.<module>.<branch>, each part escaped as a name (A-80).
BranchBuildPrefix = "branch-build."
)
// BranchBuild is what a branch-build key holds: the newest build of a module on a watched branch that was heard published,
// its source revision, the fingerprint of the branch's catalog entry at that revision and each artifact's digest.
type BranchBuild struct {
V int64 `json:"v"`
Version string `json:"version"`
Source int64 `json:"source"`
Fp string `json:"fp"`
Artifacts map[string]string `json:"artifacts"`
At time.Time `json:"at"`
}
// BranchBuildKey is the key of the state heard of a module's newest build on a branch.
func BranchBuildKey(module, branch string) string {
return BranchBuildPrefix + keys.EscapeName(module) + "." + keys.EscapeName(branch)
}
// BuildEdgesKey is the key of the state build-edges of a module's build on a branch, each part escaped as a name.
func BuildEdgesKey(module, branch string) string {
return meshcatalog.BuildEdgesByModuleBranchKey(keys.EscapeName(module), keys.EscapeName(branch))
}
// branchEntryAt is the module's entry in the newest revision of the branch taken in from that source revision (A-83: the
// revision whose source the build names, which the indexer holds itself).
func (ix *Indexer) branchEntryAt(ctx context.Context, module, branch string, source int64) (meshcatalog.CatalogEntry, error) {
if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry says the build's fingerprint")
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
number := TrunkRevisionOf
if !isTrunkBranch(branch) {
number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) }
}
var numbers []int64
for _, n := range names {
if r, ok := number(n); ok {
numbers = append(numbers, r)
}
}
slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) })
for _, n := range numbers {
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: n})
rev, entries, err := ReadRevision(ctx, ix.Revisions, name)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
if rev.Source != source {
continue
}
for _, e := range entries {
if e.Module == module {
return e, nil
}
}
}
return meshcatalog.CatalogEntry{}, fmt.Errorf("the indexer keeps no revision of the branch %s at source revision %d with an entry of %s", branch, source, module)
}
// hearBranchBuild takes the package-published of a build of a watched branch other than the first (A-83): the build is kept as
// heard, and its edge entries are kept in build-edges, keyed by the fingerprint of the entry in the revision of that branch the
// build's source names. A build of an older source revision than the one kept leaves what is kept.
func (ix *Indexer) hearBranchBuild(ctx context.Context, p meshbuilds.PackagePublished) error {
if ix.BuildEdges == nil {
return errors.New("the state build-edges is not kept here")
}
entry, err := ix.branchEntryAt(ctx, p.Module, p.Branch, p.Source)
if err != nil {
return err
}
if entry.Fp == "" {
ix.logf("build-edges: the entry of %s on %s at source revision %d has no fingerprint, so its build keeps none", p.Module, p.Branch, p.Source)
return nil
}
at := ix.now().UTC()
err = write(ctx, ix.Heard, BranchBuildKey(p.Module, p.Branch), func(raw []byte, found bool) ([]byte, bool, error) {
var b BranchBuild
if found && json.Unmarshal(raw, &b) == nil && b.Source > p.Source {
return nil, false, nil
}
if !found || b.Source != p.Source || b.Fp != entry.Fp {
b = BranchBuild{Artifacts: map[string]string{}}
}
b.V, b.Version, b.Source, b.Fp, b.At = 1, p.Version, p.Source, entry.Fp, at
b.Artifacts[p.Artifact] = p.Digest
data, err := json.Marshal(b)
return data, false, err
})
if err != nil {
return err
}
edges := releaseEdges(entry, p.Dependencies)
return write(ctx, ix.BuildEdges, BuildEdgesKey(p.Module, p.Branch), func(raw []byte, found bool) ([]byte, bool, error) {
be := meshcatalog.BuildEdges{V: 1, Fp: entry.Fp, Source: p.Source}
if found {
kept, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw)
if err != nil {
return nil, false, err
}
if kept.Source > p.Source {
return nil, false, nil
}
if kept.Source == p.Source && kept.Fp == entry.Fp {
be.Edges = kept.Edges
}
}
for _, e := range edges {
if !slices.ContainsFunc(be.Edges, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word }) {
be.Edges = append(be.Edges, e)
}
}
data, err := json.Marshal(be)
return data, false, err
})
}
// PruneBranchBuilds forgets a module's builds of the branches its registration no longer watches beside its first
// (registration-changed, §374): its build-edges keys and its branch-build keys of every other branch.
func (ix *Indexer) PruneBranchBuilds(ctx context.Context, module string, watched []string) error {
keep := map[string]bool{}
if len(watched) > 1 {
for _, b := range watched[1:] {
keep[keys.EscapeName(b)] = true
}
}
var errs []error
forget := func(st Store, key string) {
if _, rev, found, err := st.Get(ctx, key); err != nil {
errs = append(errs, err)
} else if found {
if err := st.Delete(ctx, key, rev); err != nil && !errors.Is(err, ErrConflict) {
errs = append(errs, err)
}
}
}
if ix.BuildEdges != nil {
all, err := ix.BuildEdges.Keys(ctx)
if err != nil {
return err
}
for _, k := range all {
if m, b, ok := meshcatalog.ParseBuildEdgesByModuleBranchKey(k); ok && m == keys.EscapeName(module) && !keep[b] {
forget(ix.BuildEdges, k)
}
}
}
if ix.Heard != nil {
all, err := ix.Heard.Keys(ctx)
if err != nil {
return err
}
prefix := BranchBuildPrefix + keys.EscapeName(module) + "."
for _, k := range all {
if b, ok := strings.CutPrefix(k, prefix); ok && !keep[b] {
forget(ix.Heard, k)
}
}
}
return errors.Join(errs...)
}
// outdatedBranchBuilds are the behind entries of kind 1 of the builds of other watched branches (A-83): the digest a build
// recorded for a package against the digest in the newest build of that package's module on the same branch, as heard, never a
// released digest.
func (ix *Indexer) outdatedBranchBuilds(ctx context.Context, in meshcatalog.OutdatedInput) ([]meshcatalog.Outdated, error) {
if ix.BuildEdges == nil || ix.Heard == nil {
return nil, nil
}
all, err := ix.BuildEdges.Keys(ctx)
if err != nil {
return nil, err
}
slices.Sort(all)
var out []meshcatalog.Outdated
for _, k := range all {
em, eb, ok := meshcatalog.ParseBuildEdgesByModuleBranchKey(k)
if !ok {
continue
}
module, err1 := keys.UnescapeName(em)
branch, err2 := keys.UnescapeName(eb)
if err1 != nil || err2 != nil || (in.Branch != "" && in.Branch != eb) {
continue
}
raw, _, found, err := ix.BuildEdges.Get(ctx, k)
if err != nil || !found {
continue
}
be, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw)
if err != nil {
return nil, fmt.Errorf("the key %s of the state %s: %w", k, BuildEdgesState, err)
}
for _, e := range be.Edges {
if e.Kind != 1 || (in.Dependency != "" && in.Dependency != e.Part) {
continue
}
var dep BranchBuild
raw, _, found, err := ix.Heard.Get(ctx, BranchBuildKey(strings.SplitN(e.Part, ":", 2)[0], branch))
if err != nil || !found || json.Unmarshal(raw, &dep) != nil {
continue
}
digest := dep.Artifacts[e.Part]
if digest == "" || digest == e.Digest {
continue
}
o := meshcatalog.Outdated{Module: module, Build: be.Fp, Kind: 1, Recorded: e.Version + " " + e.Digest, Newest: dep.Version + " " + digest,
Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: branch}}
if e.Stale {
o.Act = "rebuild"
}
out = append(out, o)
}
}
return out, nil
}
@@ -0,0 +1,90 @@
package index
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// Design 47 G §374, A-83, A-85: a build of a watched branch other than the first releases no version; its edge entries are kept
// in build-edges under <module>.<branch>, the fingerprint being the entry's in the branch's revision whose source the build
// names; outdated compares kind 1 with the same branch's newest build, as heard; registration-changed prunes the branch's
// keys when it is no longer watched.
func TestABuildOfAnotherWatchedBranchKeepsItsEdgesAndIsOutdatedAgainstThatBranch(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Versions: newMem(), Heard: newMem(), BuildEdges: newMem(), Revisions: newMemObjects(1 << 20), Registry: &Registry{Registrations: MemoryStore()}}
for _, m := range []string{"lib", "app"} {
raw, _ := json.Marshal(meshcatalog.Registration{Module: m, Branches: []string{"main", "beta"}})
if err := ix.Registry.Registrations.Create(ctx, RegistrationKey(m), raw); err != nil {
t.Fatal(err)
}
}
fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) }
keep := func(source int64, libFp, appFp string) {
t.Helper()
err := ix.KeepBranchEntries(ctx, "beta", source, map[string]meshcatalog.CatalogEntry{
"lib": {Module: "lib", Fp: libFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}},
"app": {Module: "app", Fp: appFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}},
Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}})
if err != nil {
t.Fatal(err)
}
}
keep(7, fp("a"), fp("b"))
keep(8, fp("c"), fp("b"))
pub := func(module, artifact, digest string, source int64, deps ...meshbuilds.Dependency) {
t.Helper()
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: module, Artifact: artifact, Digest: digest, Version: "0.1.1", Source: source, Branch: "beta", Dependencies: deps}); err != nil {
t.Fatal(err)
}
}
pub("lib", "lib:pkg", "d1", 7)
pub("app", "app:program", "a1", 7, meshbuilds.Dependency{Kind: 1, Part: "lib:pkg", Version: "0.1.1", Digest: "d1"})
if v, _ := ModuleVersions(ctx, ix.Versions, "app"); len(v) != 0 {
t.Errorf("a build of beta released %v", v)
}
raw, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta"))
be, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw)
if !found || err != nil || be.Fp != fp("b") || be.Source != 7 || len(be.Edges) != 1 || be.Edges[0].Digest != "d1" {
t.Fatalf("build-edges of app on beta: %+v (found %v, %v), want the fingerprint of the entry at source 7 and its kind 1 edge", be, found, err)
}
behind := func() []meshcatalog.Outdated {
t.Helper()
a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{})
if err != nil {
t.Fatal(err)
}
return a.Behind
}
if b := behind(); len(b) != 0 {
t.Fatalf("behind before lib's next build: %+v", b)
}
pub("lib", "lib:pkg", "d2", 8)
b := behind()
if len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Build != fp("b") || b[0].Version != "" || b[0].Dependency.Branch != "beta" ||
!strings.Contains(b[0].Recorded, "d1") || !strings.Contains(b[0].Newest, "d2") || b[0].Act != "rebuild" {
t.Fatalf("behind after lib's newer build on beta: %+v", b)
}
if err := ix.PruneBranchBuilds(ctx, "app", []string{"main", "beta"}); err != nil {
t.Fatal(err)
}
if _, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")); !found {
t.Errorf("a branch still watched lost its build-edges")
}
if err := ix.PruneBranchBuilds(ctx, "app", []string{"main"}); err != nil {
t.Fatal(err)
}
if _, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")); found {
t.Errorf("build-edges of app on beta stayed after beta was unwatched")
}
if _, _, found, _ := ix.Heard.Get(ctx, BranchBuildKey("app", "beta")); found {
t.Errorf("branch-build of app on beta stayed after beta was unwatched")
}
if _, _, found, _ := ix.Heard.Get(ctx, BranchBuildKey("lib", "beta")); !found {
t.Errorf("lib's branch-build went with app's")
}
}
@@ -0,0 +1,55 @@
package index
import (
"context"
"errors"
"maps"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// ApplyBranchSource takes a watched branch other than the first in from the source revision the seat answered for it
// (design 47 G §13, §14, §291, ADR 0666 §6, A-85, A-93): a pure computation from the merged messages of the heads the source
// names, over a view in memory seeded from the branch's newest revision, written straight into the branch's next revision
// (KeepBranchEntries): each module's fingerprint, inputs and graph node at the branch's heads, computed as the trunk's are
// (applyMoving). Nothing of the trunk is written: its modules, manifests, taken state, graph and packages stay its own,
// and the branch has no bucket of its own. A head whose merged message is not read keeps no revision, said; the next
// source-moved of the branch takes it in again.
func (ix *Indexer) ApplyBranchSource(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error {
if isTrunkBranch(branch) || ix.Revisions == nil || ix.Message == nil || len(a.Repositories) == 0 {
return nil
}
heads := map[string]string{}
for _, r := range a.Repositories {
heads[r.Repository] = r.Head
}
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
if errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: the branch %s is not taken in yet: its objects %s are not on the bus", branch, CatalogRevisionsObjects)
return nil
}
if err != nil {
return err
}
d := ix.dry()
// The branch's own view: its modules and graph from its revision, no taken merge, and the packages it publishes in memory
// over the trunk's, never written to them.
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
if ix.Packages != nil {
d.ix.Packages = newOverlay(ix.Packages)
}
d.ix.source = sourceNow{rev: uint64(a.Revision), heads: maps.Clone(heads)}
d.ix.Logf = func(format string, args ...any) {
ix.logf("branch "+branch+", computed and not written: "+format, args...)
}
if unread := d.ix.takeInHeads(ctx, heads); len(unread) > 0 {
ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: %d repositories were not taken in (%v); taken in again at its next source-moved", branch, a.Revision, len(unread), unread)
return nil
}
entries, err := d.ix.viewEntries(ctx)
if err != nil {
return err
}
return ix.KeepBranchEntries(ctx, branch, a.Revision, entries)
}
@@ -0,0 +1,75 @@
package index
import (
"context"
"encoding/json"
"fmt"
"maps"
"slices"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
)
// Design 47 G §13, §14, §291, A-93: a watched branch other than the first is taken in as a pure computation from the merged
// messages of its heads, straight into branch.<branch>.<n>; the trunk's modules, graph and revision stay the trunk's.
func TestABranchOtherThanTheFirstIsTakenInIntoItsOwnRevisionAndTheTrunkIsLeftAlone(t *testing.T) {
ctx := context.Background()
ix, mods, merged := newFormIndexerUnmerged(t, builderModules(t)...)
ix.Revisions = newMemObjects(0)
ix.Graph = newMem()
ix.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) {
if repository != catalogueRepo || head != merged.Merge {
return nil, fmt.Errorf("no merge of %s at %s", repository, head)
}
return json.Marshal(merged)
}
var said []meshcatalog.CatalogRevisionEvent
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
said = append(said, e)
return nil
}
before := map[string]string{}
keys, _ := mods.Keys(ctx)
for _, k := range keys {
v, _, _, _ := mods.Get(ctx, k)
before[k] = string(v)
}
graphBefore, _ := ix.Graph.Keys(ctx)
answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}}
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil {
t.Fatal(err)
}
if len(said) != 1 || said[0].Revision != 1 || said[0].Branch != "beta" || said[0].Source != 9 || said[0].Entries != int64(len(builderModules(t))) {
t.Fatalf("the branch take-in said %+v, want revision 1 of beta at source 9 with every module", said)
}
rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1))
if err != nil || rev.Branch != "beta" || len(entries) != len(builderModules(t)) {
t.Fatalf("branch.beta.1 = %+v with %d entries (%v)", rev, len(entries), err)
}
for _, e := range entries {
if e.Commit != merged.Merge || e.Inputs == nil { // computed at the branch's head, with the inputs its fingerprint stands on
t.Errorf("the entry of %s is at %q with inputs %v, want the branch's head %s with inputs", e.Module, e.Commit, e.Inputs, merged.Merge)
}
}
if _, any, _ := newestTrunk(ctx, ix.Revisions); any {
t.Error("a branch take-in kept a trunk revision")
}
after := map[string]string{}
keys, _ = mods.Keys(ctx)
for _, k := range keys {
v, _, _, _ := mods.Get(ctx, k)
after[k] = string(v)
}
if !maps.Equal(before, after) {
t.Errorf("a branch take-in wrote the trunk's modules")
}
if graphAfter, _ := ix.Graph.Keys(ctx); !slices.Equal(graphBefore, graphAfter) {
t.Errorf("a branch take-in wrote the trunk's graph: %v, was %v", graphAfter, graphBefore)
}
// Taken in again at the same heads, nothing moved: no second revision.
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil || len(said) != 1 {
t.Errorf("the same heads said %d revisions (%v), want one", len(said), err)
}
}
@@ -110,6 +110,17 @@ type Indexer struct {
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
// id given (W4.6).
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
// Versions is the state versions (versions.go, design 47 G §349): each module's released versions; nil keeps none.
Versions Store
// BuildEdges is the state build-edges (branch_builds.go, design 47 G §374): the edge entries of the newest build of a module on a
// watched branch other than the first; nil keeps none.
BuildEdges Store
// SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing.
SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error
// SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing.
SayVersionsFilled func(ctx context.Context, e meshcatalog.VersionsFilledEvent, id string) error
// Inputs is the state inputs (run_inputs.go, design 47 G §354): the inputs of the indexer's run asks; nil keeps none.
Inputs Store
// Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads
// what the first registrations are composed from (§8). Nil: no registration act is answered.
Registry *Registry
@@ -147,26 +158,28 @@ type Indexer struct {
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
freshMu sync.Mutex
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
freshMu sync.Mutex
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
OwnBuild func(ctx context.Context) (string, error)
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
@@ -183,7 +196,15 @@ type Indexer struct {
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
// toolchain.<language>, delivered.<module>, pending.<module>.<version>, acted.<acting part>.<dependent>, each key part escaped as a name
// (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled.
Heard Store
// Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the
// edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered".
Placement PlacementAsker
RunningWhole RunningReader
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
// Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
Message func(ctx context.Context, repository, merge string, number int) ([]byte, error)
@@ -0,0 +1,66 @@
package index
import (
"context"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// Design 47 G A-85, §291: the indexer keeps a catalog revision per watched branch, trunk.<n> for the first and
// branch.<branch>.<n> (the branch escaped) for another, each numbered by its own branch, and the verb revision answers the
// branch it is asked for; the event names the branch, escaped.
func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Revisions: newMemObjects(1 << 20), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
var said []meshcatalog.CatalogRevisionEvent
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil }
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Commit: "c1", Invokes: []string{RevisionInvoke}, Manifest: map[string]any{}}
tip := func(commit string) map[string]meshcatalog.CatalogEntry {
e := reader
e.Commit = commit
return map[string]meshcatalog.CatalogEntry{"reader": e}
}
if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1")); err != nil {
t.Fatal(err)
}
if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2")); err != nil {
t.Fatal(err)
}
if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3")); err != nil {
t.Fatal(err)
}
names, _ := ix.Revisions.Names(ctx)
have := map[string]bool{}
for _, n := range names {
have[n] = true
}
for _, want := range []string{"trunk.1", "branch.feature/x=2ey.1", "branch.feature/x=2ey.2"} {
if !have[want] {
t.Errorf("the objects hold %v, want %s", names, want)
}
}
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 {
t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said)
}
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
if err != nil {
t.Fatalf("revision of %q: %v", branch, err)
}
return a
}
if a := answer("main", 0); a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" {
t.Errorf("main answered %+v", a)
}
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision)
}
if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 1 answered %+v", a.Revision)
}
if a := answer("beta", 0); !a.NoneYet {
t.Errorf("a branch with no revision answered %+v, want none-yet", a)
}
}
@@ -14,6 +14,7 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -46,15 +47,41 @@ type Objects interface {
// EntryObject is the object name of the entry of that digest.
func EntryObject(digest string) string { return "entry." + digest }
// RevisionObject is the object name of a revision: trunk.<revision>, or candidate.<group>.<heads> for a candidate,
// TrunkBranch is the first watched branch, whose revisions are kept as trunk.<revision> (design 47 G §291, A-85).
const TrunkBranch = "main"
// isTrunkBranch says whether a branch's revisions are the trunk's: the first watched branch, or none named.
func isTrunkBranch(branch string) bool { return branch == "" || branch == TrunkBranch }
// BranchObject is the object name of a revision of a watched branch other than the first: branch.<branch>.<revision>,
// the branch escaped as a name (A-85), so a dot in it never reads as a part.
func BranchObject(branch string, revision int64) string {
return "branch." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
}
// RevisionObject is the object name of a revision: trunk.<revision> for the first watched branch,
// branch.<branch>.<revision> for another watched branch (A-85), or candidate.<group>.<heads> for a candidate,
// which keeps its newest revision only.
func RevisionObject(rev meshcatalog.CatalogRevision) string {
if rev.Kind == "candidate" {
return "candidate." + rev.Group + "." + rev.Heads
}
if !isTrunkBranch(rev.Branch) {
return BranchObject(rev.Branch, rev.Revision)
}
return "trunk." + strconv.FormatInt(rev.Revision, 10)
}
// BranchRevisionOf is the revision number a branch object's name holds for that branch, and whether it is one.
func BranchRevisionOf(branch, name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "branch."+keys.EscapeName(branch)+".")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil
}
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
func TrunkRevisionOf(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
@@ -128,26 +155,37 @@ func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.Cata
// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false.
func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) {
return newestOf(ctx, o, "")
}
// newestOf is the newest revision of a watched branch the bucket keeps (the trunk's for the first watched branch or none
// named), its list only; none is a zero revision and false.
func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshcatalog.CatalogRevision{}, false, err
}
number := TrunkRevisionOf
if !isTrunkBranch(branch) {
number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) }
}
var newest int64
for _, n := range names {
if r, ok := TrunkRevisionOf(n); ok && r > newest {
if r, ok := number(n); ok && r > newest {
newest = r
}
}
if newest == 0 {
return meshcatalog.CatalogRevision{}, false, nil
}
data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10))
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest})
data, err := o.Get(ctx, name)
if err != nil {
return meshcatalog.CatalogRevision{}, false, err
}
var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil {
return rev, false, fmt.Errorf("trunk.%d: %w", newest, err)
return rev, false, fmt.Errorf("%s: %w", name, err)
}
return rev, true, nil
}
@@ -208,15 +246,25 @@ func (ix *Indexer) KeepView(ctx context.Context) error {
if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() {
return nil
}
held, err := ix.heldModules(ctx)
entries, err := ix.viewEntries(ctx)
if err != nil {
return err
}
return ix.keepEntries(ctx, entries)
}
// viewEntries is the entry of every module the indexer's view (its modules and its module graph) holds: the module's value
// with its node of the graph, as KeepView keeps them.
func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.CatalogEntry, error) {
held, err := ix.heldModules(ctx)
if err != nil {
return nil, err
}
nodes := graph.Graph{}
if ix.Graph != nil {
keys, err := ix.Graph.Keys(ctx)
if err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the module graph: %w", err)
return nil, fmt.Errorf("the module graph: %w", err)
}
for _, k := range keys {
if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found {
@@ -237,7 +285,7 @@ func (ix *Indexer) KeepView(ctx context.Context) error {
}
entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, m, raw, nodes[name]), m)
}
return ix.keepEntries(ctx, entries)
return entries, nil
}
// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry
@@ -246,12 +294,26 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
if ix.catchingUp.Load() {
return nil // the catch-up of a first start keeps its one revision at its end (CatchUp)
}
ix.keepMu.Lock()
defer ix.keepMu.Unlock()
ix.mu.Lock()
src := int64(ix.source.rev)
ix.mu.Unlock()
newest, any, err := newestTrunk(ctx, ix.Revisions)
return ix.keepBranchEntries(ctx, TrunkBranch, src, entries)
}
// KeepBranchEntries keeps entries as the revision of a watched branch after that branch's newest (design 47 G §291,
// A-85): trunk.<revision> for the first watched branch, branch.<branch>.<revision> for another, numbered per branch, and
// says catalog-revision naming the branch. src is the branch's source revision the entries were taken in from.
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error {
return ix.keepBranchEntries(ctx, branch, src, entries)
}
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error {
if branch == "" {
branch = TrunkBranch
}
ix.keepMu.Lock()
defer ix.keepMu.Unlock()
newest, any, err := newestOf(ctx, ix.Revisions, branch)
if errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
return nil
@@ -261,7 +323,7 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
return fmt.Errorf("the newest catalog graph revision: %w", err)
}
at := ix.now().UTC()
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries)
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries)
if err != nil {
return err
}
@@ -295,7 +357,7 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
if ix.SayCatalogRevision != nil {
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
}
}
@@ -362,8 +424,14 @@ func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's
// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view.
func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) {
return BranchView(ctx, o, TrunkBranch)
}
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
// (design 47 G §291, A-93), empty and at 0 when it keeps none.
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
view, held := newOverlay(nil), newOverlay(nil)
newest, any, err := newestTrunk(ctx, o)
newest, any, err := newestOf(ctx, o, branch)
if err != nil || !any {
return view, held, 0, err
}
@@ -908,9 +908,9 @@ func (ix *Indexer) dry() *dryRun {
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
OwnBuildState: nil, underSeats: true,
OwnBuildState: nil, Heard: nil, Placement: nil, RunningWhole: nil, underSeats: true,
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil,
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil,
// A dry run writes no registration and asks no one.
Registry: nil, SeedModules: nil}
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
@@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged",
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules"} {
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges"} {
named[f] = true
}
ty := reflect.TypeOf(Indexer{})
@@ -122,6 +122,9 @@ type Registry struct {
HistoryOf func(ctx context.Context, module string) ([]meshcatalog.Registration, error)
// Say says the seat mesh-catalog's event registration-changed.
Say func(ctx context.Context, e meshcatalog.RegistrationChangedEvent) error
// Pruned is told after a registration changed which branches the module watches, so what is kept of its builds of a branch
// it no longer watches goes (registration-changed, design 47 G §374); nil prunes none.
Pruned func(ctx context.Context, module string, watched []string) error
// Branch reads a branch through the seat git; an empty branch asks the repository's default branch.
Branch func(repository, branch string) (gitseat.BranchAnswer, error)
// Placement reads a module's placements through the seat mesh-module-assigner's verb placement: one per watched branch a row
@@ -816,6 +819,11 @@ func (r *Registry) write(ctx context.Context, key string, p Proposal, next meshc
r.logf("registration-changed of %s %d is not said: %v", next.Module, next.Version, err)
}
}
if r.Pruned != nil {
if err := r.Pruned(ctx, next.Module, next.Branches); err != nil {
r.logf("the builds of %s on the branches it no longer watches are not pruned: %v", next.Module, err)
}
}
return r.endKey(ctx, key, p, ProposalApplied, fmt.Sprintf("applied as version %d of %s on %s", next.Version, next.Module, occ.Says(w)))
}
@@ -5,11 +5,11 @@ import (
"encoding/json"
"fmt"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -53,6 +53,22 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
}
// A watched branch other than the first keeps its own revisions (A-85); the first's are the trunk's. The branch is
// named as a key name is escaped, so it is read back to the branch's own name.
branch, err := keys.UnescapeName(in.Branch)
if err != nil && in.Branch != "" {
return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err)
}
if !isTrunkBranch(branch) {
other, found, err := newestOf(ctx, o, branch)
if err != nil {
return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err)
}
if !found {
return meshcatalog.RevisionAnswer{NoneYet: true}, nil
}
newest = other
}
name := RevisionObject(newest)
switch {
case in.Group != "" || in.Heads != "":
@@ -61,10 +77,14 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata
}
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads})
case in.Revision != 0 && in.Revision != newest.Revision:
name = "trunk." + strconv.FormatInt(in.Revision, 10)
name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: in.Revision})
}
rev, entries := newest, trunkEntries
if name != RevisionObject(newest) {
if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" {
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return out, err
}
} else if name != RevisionObject(newest) {
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return out, err
}
@@ -0,0 +1,233 @@
package index
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The indexer's inputs (design 47 G §354): the input of the ask `run` the indexer's validation puts, kept in the old state
// section inputs, one key per input named by its write time and its sha256, read by the seat mesh-catalog's verb input (whose
// callers are the holders of the build seats) and by the tool indexer_inputs. A key older than a day is deleted at each new
// ask, judged over at most the 20 oldest keys.
const (
InputsState = "inputs"
InputsBucket = Module + "_" + InputsState
InputsTool = "indexer_inputs"
// InputsKeptFor is how long an input is kept at the longest.
InputsKeptFor = 24 * time.Hour
// InputsSwept is how many of the oldest keys each new ask judges.
InputsSwept = 20
inputsTime = "20060102T150405Z"
// InputsAskPrefix opens the keys that name an input by the id of the run ask it was kept for (A-78 d): ask.<id>, the id escaped as a
// name (§73), holding the input's own key, so the build seat's step-ended of that ask finds it and deletes it.
InputsAskPrefix = "ask."
)
// InputDigest is the sha256 of an input's exact bytes, as "sha256:" and its hex digits.
func InputDigest(body []byte) string {
sum := sha256.Sum256(body)
return "sha256:" + hex.EncodeToString(sum[:])
}
// KeepInput keeps the exact bytes of the input of the run ask ask under their write time and sha256, after sweeping the keys more
// than a day old among the 20 oldest, and names it by the ask's id (A-78 d); it answers the digest. The same bytes written again in
// the same second are one key.
func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the run ask the input is kept for")
}
if ix.Inputs == nil {
return "", errors.New("the state inputs is not kept here")
}
if err := ix.sweepInputs(ctx); err != nil {
ix.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
digest := InputDigest(body)
key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest)
data, err := json.Marshal(base64.StdEncoding.EncodeToString(body))
if err != nil {
return "", err
}
if err := ix.Inputs.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
return "", err
}
if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) {
return "", err
}
return digest, nil
}
// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask
// that kept none, or whose input a sweep deleted, changes nothing.
func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error {
if ix.Inputs == nil || e.Ask == "" {
return nil
}
askKey := InputsAskPrefix + keys.EscapeName(e.Ask)
raw, rev, found, err := ix.Inputs.Get(ctx, askKey)
if err != nil || !found {
return err
}
if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil {
return err
} else if ok {
if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil {
return err
}
}
return ix.Inputs.Delete(ctx, askKey, rev)
}
// sweepInputs deletes, among the 20 oldest keys, those written more than a day ago.
func (ix *Indexer) sweepInputs(ctx context.Context) error {
all, err := ix.Inputs.Keys(ctx)
if err != nil {
return err
}
var errs []error
var inputs, asks []string
for _, k := range all {
if strings.HasPrefix(k, InputsAskPrefix) {
asks = append(asks, k)
} else {
inputs = append(inputs, k)
}
}
slices.Sort(inputs) // the write time leads the key, so the oldest come first
if len(inputs) > InputsSwept {
inputs = inputs[:InputsSwept]
}
for _, k := range inputs {
t, _, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor {
continue
}
if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
errs = append(errs, err)
}
}
}
// a name by ask whose input a sweep or its step-ended deleted goes with it
for _, k := range asks {
if raw, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
if _, _, ok, err := ix.Inputs.Get(ctx, string(raw)); err == nil && !ok {
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
errs = append(errs, err)
}
}
}
}
return errors.Join(errs...)
}
// InputKey is the key of the input of the digest, or none.
func InputKey(ctx context.Context, st Store, digest string) (string, error) {
all, err := st.Keys(ctx)
if err != nil {
return "", err
}
for _, k := range all {
if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest && !strings.HasPrefix(k, InputsAskPrefix) {
return k, nil
}
}
return "", nil
}
// Input answers the verb input: the exact bytes (base64) of the input of the digest, its digest checked.
func Input(ctx context.Context, st Store, in meshcatalog.InputInput) (meshcatalog.InputAnswer, error) {
var out meshcatalog.InputAnswer
if in.Digest == "" {
return out, errors.New("name the input's sha256 (digest)")
}
key, err := InputKey(ctx, st, in.Digest)
if err != nil {
return out, fmt.Errorf("the state %s cannot be read: %w", InputsState, err)
}
raw, _, found, err := st.Get(ctx, key)
if key == "" || err != nil || !found {
return out, fmt.Errorf("no input of digest %s is kept", in.Digest)
}
text, err := meshcatalog.DecodeInputsByTimeDigestValue(raw)
if err != nil {
return out, fmt.Errorf("the input %s cannot be read: %w", key, err)
}
body, err := base64.StdEncoding.DecodeString(text)
if err != nil || InputDigest(body) != in.Digest {
return out, fmt.Errorf("the input kept under %s does not match its digest", key)
}
return meshcatalog.InputAnswer{Digest: in.Digest, Input: text}, nil
}
// InputTool is the verb input as the tool mesh-catalog.input of the indexer's own runtime.
func InputTool(st Store) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbInput, Run: func(args map[string]any) (any, error) {
var in meshcatalog.InputInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Input(context.Background(), st, in)
}}
}
// InputsAnswer is what indexer_inputs answers: each key kept with its time, digest and size, or one input's bytes.
type InputsAnswer struct {
Inputs []InputKept `json:"inputs"`
}
// InputKept is one input kept.
type InputKept struct {
Key string `json:"key"`
Digest string `json:"digest"`
Bytes int `json:"bytes"`
Input string `json:"input,omitempty"`
}
// InputsReadTool is indexer_inputs (class read): every input the indexer keeps, with its key, digest and size; with a digest,
// that input whole (base64).
func InputsReadTool(st Store) stdio.Tool {
return stdio.Tool{Name: InputsTool, Description: "Read the indexer's inputs of the ask run (design 47 G section 354): each key (write time and sha256) with its size; with a digest, that input's exact bytes, base64.",
Input: map[string]any{"type": "object", "properties": map[string]any{"digest": map[string]any{"type": "string", "description": "one input's sha256; none lists every key"}}},
Run: func(args map[string]any) (any, error) {
ctx := context.Background()
want, _ := args["digest"].(string)
all, err := st.Keys(ctx)
if err != nil {
return nil, err
}
slices.Sort(all)
out := InputsAnswer{Inputs: []InputKept{}}
for _, k := range all {
_, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
if !ok || strings.HasPrefix(k, InputsAskPrefix) || (want != "" && d != want) {
continue
}
raw, _, found, err := st.Get(ctx, k)
if err != nil || !found {
continue
}
text, _ := meshcatalog.DecodeInputsByTimeDigestValue(raw)
kept := InputKept{Key: k, Digest: d, Bytes: base64.StdEncoding.DecodedLen(len(text))}
if want != "" {
kept.Input = text
}
out.Inputs = append(out.Inputs, kept)
}
return out, nil
}}
}
@@ -0,0 +1,227 @@
package index
import (
"context"
"fmt"
"maps"
"path"
"slices"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// StepsVerb is the verb of each build seat the seed asks (design 47 G §704).
const StepsVerb = "steps"
// StepsCaller asks the build seat seat its verb steps (a kept call, ADR 0668 §1): the answer, or why none came.
type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error)
// ModuleManifest is the file name of a module's manifest in its directory.
const ModuleManifest = "module.json"
// buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704,
// §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said.
func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) {
var seats []string
for _, name := range slices.Sorted(maps.Keys(defs)) {
for _, r := range defs[name].Acts {
if r.Step != seatdef.StepBuild {
continue
}
if r.ActsOn(a.Language) || (r.On == seatdef.OnKindImage && a.Kind == "image") {
seats = append(seats, name)
break
}
}
}
switch len(seats) {
case 0:
return "", fmt.Errorf("no seat builds the artifact %s (language %q, kind %q)", a.Ref, a.Language, a.Kind)
case 1:
return seats[0], nil
}
return "", fmt.Errorf("the seats %v each build the artifact %s", seats, a.Ref)
}
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
// it is unanswered five minutes and cleared when it answers.
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
if ix.Revisions == nil {
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return nil, err
}
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
for _, e := range entries {
for _, a := range e.Artifacts {
seat, err := buildSeatOf(defs, a)
if err != nil {
return nil, fmt.Errorf("%s: %w", e.Module, err)
}
seatOf[a.Ref] = seat
}
}
answered := map[string]bool{}
unanswered := map[string]error{}
var out []SeedModule
for _, e := range entries {
m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)}
for _, a := range e.Artifacts {
seat := seatOf[a.Ref]
name := artifactName(a.Ref)
sa := SeedArtifact{Name: name, Seat: seat}
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name), Packages: maps.Clone(e.Packages)}}
ans, err := steps(ctx, seat, in)
switch {
case err != nil:
unanswered[seat] = err
case ans.Error != "":
unanswered[seat] = fmt.Errorf("%s", ans.Error)
default:
sa.Answered = true
answered[seat] = true
for _, s := range ans.Steps {
sa.Steps = append(sa.Steps, s.Step)
}
}
m.Artifacts = append(m.Artifacts, sa)
}
out = append(out, m)
}
ix.stepsFindings(ctx, answered, unanswered)
return out, nil
}
// artifactName is the artifact's name in an artifact reference <module>:<artifact>.
func artifactName(ref string) string {
for i := len(ref) - 1; i >= 0; i-- {
if ref[i] == ':' {
return ref[i+1:]
}
}
return ref
}
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
if ix.Findings == nil {
return
}
now := ix.now()
var raise []string
since := map[string]time.Time{}
for seat := range unanswered {
since[seat] = ix.stepsUnansweredSince(ctx, seat, now)
if now.Sub(since[seat]) >= mcc.CallUnansweredAfter {
raise = append(raise, seat)
}
}
for seat := range answered {
if _, bad := unanswered[seat]; !bad {
ix.stepsAnswered(ctx, seat)
}
}
for _, seat := range slices.Sorted(slices.Values(raise)) {
id := seat + "." + StepsVerb
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
"A call whose answer is kept is not answered",
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
"indexer."+mcc.KindCallUnanswered)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("seed: the finding %s is not written: %v", id, err)
}
}
for seat := range answered {
if _, bad := unanswered[seat]; bad {
continue
}
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
}
}
}
// namedPaths are the named paths the entry keeps for the artifact, by its name or its reference.
func namedPaths(e meshcatalog.CatalogEntry, name string) []string {
if e.Inputs == nil {
return nil
}
if p, ok := e.Inputs.Paths[name]; ok {
return p
}
return e.Inputs.Paths[ArtifactRefOf(e.Module, name)]
}
// StepsSincePrefix opens the keys of the state heard (Heard, A-80) that keep, by build seat, since when its steps
// has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past
// five minutes.
const StepsSincePrefix = "steps-since."
// stepsUnansweredSince is the time the seat first failed to answer.
func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time {
return ix.unansweredSince(ctx, StepsSincePrefix+keys.EscapeName(seat), now)
}
// stepsAnswered forgets the time a seat that answers was unanswered since.
func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) {
ix.answered(ctx, StepsSincePrefix+keys.EscapeName(seat))
}
// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's state heard (the first
// failure writes now); where no state is kept it is held in memory.
func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time {
if ix.Heard == nil {
ix.mu.Lock()
defer ix.mu.Unlock()
if ix.stepsSince == nil {
ix.stepsSince = map[string]time.Time{}
}
if _, ok := ix.stepsSince[key]; !ok {
ix.stepsSince[key] = now
}
return ix.stepsSince[key]
}
if v, _, found, err := ix.Heard.Get(ctx, key); err == nil && found {
if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil {
return t
}
}
if err := ix.Heard.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil {
ix.logf("the time since %s is unanswered is not kept: %v", key, err)
}
return now
}
// answered forgets the time the call kept at key was unanswered since.
func (ix *Indexer) answered(ctx context.Context, key string) {
if ix.Heard == nil {
ix.mu.Lock()
delete(ix.stepsSince, key)
ix.mu.Unlock()
return
}
if _, rev, found, err := ix.Heard.Get(ctx, key); err == nil && found {
if err := ix.Heard.Delete(ctx, key, rev); err != nil {
ix.logf("the time since %s is unanswered is not forgotten: %v", key, err)
}
}
}
@@ -0,0 +1,95 @@
package index
import (
"context"
"errors"
"strings"
"testing"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// Design 47 G §704: the seed asks `steps` of the build seat whose `build` rule matches each artifact, with {module,
// artifact, fingerprint, fetch} from the newest trunk entry; a seat that does not answer holds the seed and, after five
// minutes, is the finding call-unanswered (scope call, id <seat>.steps), cleared when it answers.
func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
w := newFindings()
ix := &Indexer{Findings: w, Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }}
fp := "src2:" + strings.Repeat("a", 64)
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{},
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}},
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"},
Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
defs := map[string]seatdef.Definition{
"node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}},
"node-build-image": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: seatdef.OnKindImage}}},
}
// the image artifact is built by both rules: the language's and the kind's
if _, err := ix.SeedModulesFrom(ctx, defs, nil); err == nil || !strings.Contains(err.Error(), "each build") {
t.Fatalf("two seats building one artifact: %v", err)
}
e := entries["avahi"]
e.Artifacts[1].Language = "dockerfile"
entries["avahi"] = e
rev, objects, _ = CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 4, Kind: "trunk"}, entries)
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
var got []nodebuildgo.StepsInput
goUp := false
call := func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) {
got = append(got, in)
if seat == "node-build-go" && !goUp {
return nodebuildgo.StepsAnswer{}, errors.New("no responders")
}
return nodebuildgo.StepsAnswer{Steps: []nodebuildgo.DetectedStep{{Step: "unit"}}}, nil
}
mods, err := ix.SeedModulesFrom(ctx, defs, call)
if err != nil {
t.Fatal(err)
}
if len(mods) != 1 || mods[0].Module != "avahi" || mods[0].Origin != "novox/r" || mods[0].Path != "mods/avahi/module.json" || len(mods[0].Artifacts) != 2 {
t.Fatalf("the seed's modules: %+v", mods)
}
if a := mods[0].Artifacts[0]; a.Answered || a.Seat != "node-build-go" {
t.Errorf("the go artifact is %+v, want unanswered by node-build-go", a)
}
if a := mods[0].Artifacts[1]; !a.Answered || a.Seat != "node-build-image" || len(a.Steps) != 1 {
t.Errorf("the image artifact is %+v", a)
}
in := got[0]
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" {
t.Errorf("the steps input is %+v", in)
}
if len(w.Held()) != 0 {
t.Fatal("a call unanswered for no time wrote its finding")
}
now = now.Add(5 * time.Minute)
ix = &Indexer{Findings: w, Heard: ix.Heard, Revisions: ix.Revisions, Now: func() time.Time { return now }} // a restart keeps the time
if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil {
t.Fatal(err)
}
if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != mcc.KindCallUnanswered {
t.Fatalf("the findings after five minutes: %+v", h)
}
goUp = true
if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil {
t.Fatal(err)
}
if len(w.Held()) != 0 {
t.Errorf("the finding stands after the seat answered: %+v", w.Held())
}
}
@@ -0,0 +1,324 @@
package index
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
// released versions in the state versions, one key per module and version (<module>.<version>) holding each artifact's digest,
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
// released once every artifact the module's entry names is published at it.
const (
VersionsState = "versions"
VersionsBucket = Module + "_" + VersionsState
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
// go, never its highest of any MAJOR.MINOR.
VersionsKept = 100
// PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the
// artifacts published at it so far, each key part escaped as a name (A-80, §73).
PendingPrefix = "pending."
)
// published is what a not yet released version holds: each artifact published at it, by reference.
type published struct {
Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"`
Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"`
Source int64 `json:"source,omitempty"`
}
// versionParts is MAJOR.MINOR.PATCH, read; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two versions MAJOR.MINOR.PATCH numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
// ModuleVersions reads every released version the state keeps for module, newest first.
func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) {
all, err := st.Keys(ctx)
if err != nil {
return nil, err
}
var out []meshcatalog.VersionEntry
for _, k := range all {
m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
if !ok || m != module {
continue
}
raw, _, found, err := st.Get(ctx, k)
if err != nil {
return nil, err
}
if !found {
continue
}
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
if err != nil {
return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err)
}
out = append(out, v)
}
slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) })
return out, nil
}
// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its
// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded,
// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A
// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83).
func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if _, ok := versionParts(p.Version); !ok {
return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version)
}
if ix.Registry != nil {
if r, found, err := ix.Registry.Registration(ctx, p.Module); err == nil && found && len(r.Branches) > 0 && r.Branches[0] != p.Branch {
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
}
}
if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil || found {
return err // released already: heard again
}
entry, err := ix.entryOf(ctx, p.Module)
if err != nil {
return err
}
var want []string
for _, a := range entry.Artifacts {
want = append(want, a.Ref)
}
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
var have published
err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) {
have = published{}
if found {
if err := json.Unmarshal(raw, &have); err != nil {
return nil, false, err
}
}
if have.Artifacts == nil {
have.Artifacts = map[string]meshcatalog.VersionArtifact{}
}
have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest}
have.Source = p.Source
for _, d := range p.Dependencies {
if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) {
have.Dependencies = append(have.Dependencies, d)
}
}
data, err := json.Marshal(have)
return data, false, err
})
if err != nil {
return err
}
for _, ref := range want {
if _, ok := have.Artifacts[ref]; !ok {
return nil
}
}
return ix.releaseVersion(ctx, p, key, entry, have)
}
// entryOf is the module's entry in the newest trunk revision.
func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) {
if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
if !found {
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
for _, e := range entries {
if e.Module == module {
return e, nil
}
}
return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module)
}
// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions,
// and forgets the artifacts kept as published.
func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error {
key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)
kept, err := ModuleVersions(ctx, ix.Versions, p.Module)
if err != nil {
return err
}
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
at := ix.now().UTC()
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
for _, a := range entry.Artifacts {
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
}
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
v.Edges = releaseEdges(entry, have.Dependencies)
data, err := json.Marshal(v)
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
return err
}
if !late && ix.SayVersionReleased != nil {
ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at}
if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil {
return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err)
}
}
if err := ix.trimVersions(ctx, p.Module); err != nil {
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
}
return ix.forgetPublished(ctx, pendingKey)
}
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
if err != nil || len(kept) <= VersionsKept {
return err
}
highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first
seen := map[string]bool{}
for _, v := range kept {
p := strings.Split(v.Version, ".")
if mm := p[0] + "." + p[1]; !seen[mm] {
seen[mm] = true
highest[v.Version] = true
}
}
excess := len(kept) - VersionsKept
for i := len(kept) - 1; i >= 0 && excess > 0; i-- {
v := kept[i].Version
if highest[v] {
continue
}
key := meshcatalog.VersionsByModuleVersionKey(module, v)
if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found {
if err := ix.Versions.Delete(ctx, key, rev); err != nil {
return err
}
}
excess--
}
return nil
}
// forgetPublished removes the artifacts kept as published at a version released.
func (ix *Indexer) forgetPublished(ctx context.Context, key string) error {
if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found {
return err
} else {
return ix.Heard.Delete(ctx, key, rev)
}
}
// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first.
func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) {
if in.Module == "" {
return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)")
}
v, err := ModuleVersions(ctx, st, in.Module)
if v == nil {
v = []meshcatalog.VersionEntry{}
}
return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err
}
// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime.
func VersionsTool(st Store) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) {
var in meshcatalog.VersionsInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Versions(context.Background(), st, in)
}}
}
// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374).
const notYetDelivered = "not yet delivered"
// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build
// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's
// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here.
func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge {
word := func(kind int64, part string) string {
for _, e := range entry.Edges {
if e.Kind == kind && (e.Part == part || e.From == part) {
return e.Word
}
}
return ""
}
var out []meshcatalog.VersionEdge
for _, d := range deps {
out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)})
}
for _, e := range entry.Edges {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
part := cmp.Or(e.Part, e.From)
if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) {
out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered})
}
}
return out
}
// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions
// state and the heard toolchain and delivered keys of the own state.
func OutdatedTool(ix *Indexer) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) {
var in meshcatalog.OutdatedInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return ix.Outdated(context.Background(), in)
}}
}
@@ -0,0 +1,381 @@
package index
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// What outdated compares against is kept as heard, in the indexer's state heard (design 47 G A-68): toolchain.<language> the newest
// toolchain-agreed per language, delivered.<module> the newest version-delivered per module.
const (
ToolchainPrefix = "toolchain."
DeliveredPrefix = "delivered."
// ActedPrefix opens acted.<acting part>.<dependent>, the newest wait-ended of an acting part's wait for a dependent, ran or released
// (A-78 a), each key part escaped as a name (A-80, §73).
ActedPrefix = "acted."
)
// Acted is what an acted key holds: the build the dependent was at when the acting part acted for it (the wait's version, §374 kind 5).
type Acted struct {
V int64 `json:"v"`
Part string `json:"part"`
Step string `json:"step"`
Build string `json:"build"`
How string `json:"how"`
At time.Time `json:"at"`
}
// HearWaitEnded keeps, for kind 5 (§374, §375, A-78 a), the newest wait-ended of an acting step that ended ran or released: the
// acting part, the step and the build it acted for. A wait that ended superseded, or of a step that is no acting part's (put,
// delivery, kept-build, backup), is not an act and keeps nothing.
func (ix *Indexer) HearWaitEnded(ctx context.Context, w meshwants.WaitEndedEvent) error {
if w.Part == "" || w.Module == "" || (w.How != "ran" && w.How != "released") {
return nil
}
switch w.Step {
case meshwants.WaitStepPut, meshwants.WaitStepDelivery, meshwants.WaitStepKeptBuild, meshwants.WaitStepBackup, "":
return nil
}
if ix.Heard == nil {
return errors.New("the state heard is not kept here")
}
a := Acted{V: 1, Part: w.Part, Step: w.Step, Build: w.Version, How: w.How, At: w.Since}
return ix.keepNewest(ctx, ActedKey(w.Part, w.Module), a, func(kept []byte) bool {
var k Acted
return json.Unmarshal(kept, &k) != nil || !a.At.Before(k.At)
})
}
// ActedKey is the key of the state heard of an acting part's act for a dependent.
func ActedKey(part, dependent string) string {
return ActedPrefix + keys.EscapeName(part) + "." + keys.EscapeName(dependent)
}
// actedFor is the act kept for the acting part (an artifact reference, or its module) and the dependent module.
func (ix *Indexer) actedFor(ctx context.Context, part, dependent string) (Acted, bool) {
for _, p := range []string{part, strings.SplitN(part, ":", 2)[0]} {
raw, _, found, err := ix.Heard.Get(ctx, ActedKey(p, dependent))
var a Acted
if err == nil && found && json.Unmarshal(raw, &a) == nil {
return a, true
}
}
return Acted{}, false
}
// keepNewest writes body at key of the state heard unless it holds a value whose version is higher; version of the body
// is read by newer, which says whether the heard one is not older than the kept.
func (ix *Indexer) keepNewest(ctx context.Context, key string, body any, newer func(kept []byte) bool) error {
if ix.Heard == nil {
return errors.New("the own state is not kept here")
}
data, err := json.Marshal(body)
if err != nil {
return err
}
return write(ctx, ix.Heard, key, func(kept []byte, found bool) ([]byte, bool, error) {
if found && !newer(kept) {
return nil, false, nil
}
return data, false, nil
})
}
// HearToolchainAgreed keeps the newest toolchain-agreed of a language (A-68, §375): the one heard last.
func (ix *Indexer) HearToolchainAgreed(ctx context.Context, t meshbuilds.ToolchainAgreed) error {
if t.Language == "" {
return errors.New("a toolchain-agreed names no language")
}
return ix.keepNewest(ctx, ToolchainPrefix+keys.EscapeName(t.Language), t, func([]byte) bool { return true })
}
// HearVersionDelivered takes one version-delivered (design 47 G §374, §376, A-68): the entries of kinds 2, 4 and 6 of the
// delivered release, which read "not yet delivered", are written once with each dependency's newest delivered version and
// part digest; then the delivered version is kept as the module's newest.
func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.VersionDeliveredEvent) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if _, ok := versionParts(d.Version); !ok {
return fmt.Errorf("the version-delivered of %s names the version %q, which is no MAJOR.MINOR.PATCH", d.Module, d.Version)
}
key := meshcatalog.VersionsByModuleVersionKey(d.Module, d.Version)
err := write(ctx, ix.Versions, key, func(raw []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil // a version the catalog does not hold (a build of another branch) has no entries to write
}
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
if err != nil {
return nil, false, err
}
changed := false
for i, e := range v.Edges {
if e.Version != notYetDelivered {
continue
}
dep, ok := ix.deliveredOf(ctx, strings.SplitN(e.Part, ":", 2)[0])
if !ok {
continue // the dependency is not delivered yet: the entry reads so until this release's own delivery finds it
}
v.Edges[i].Version, v.Edges[i].Digest = dep.Version, dep.Parts[e.Part]
changed = true
}
if !changed {
return nil, false, nil
}
data, err := json.Marshal(v)
return data, false, err
})
if err != nil {
return err
}
return ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(d.Module), d, func(kept []byte) bool {
var k meshwants.VersionDeliveredEvent
return json.Unmarshal(kept, &k) != nil || compareVersions(d.Version, k.Version) >= 0
})
}
// deliveredOf is the newest version-delivered kept of a module.
func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.VersionDeliveredEvent, bool) {
var d meshwants.VersionDeliveredEvent
raw, _, found, err := ix.Heard.Get(ctx, DeliveredPrefix+keys.EscapeName(module))
if err != nil || !found || json.Unmarshal(raw, &d) != nil {
return d, false
}
return d, true
}
// Outdated answers the verb outdated (design 47 G §375) for every edge kind: every entry of each module's newest released
// version that is behind. Kind 1: the recorded digest differs from the part's digest in the dependency's newest released
// version. Kind 3: the recorded toolchain version differs from the version of the newest toolchain-agreed of the dependent
// artifact's language (A-80, toolchain.<language>). Kinds 2 and 4: from the dependency's newest version-delivered. Kind 5: the
// acting part has not acted for the dependent's current build (acted.<acting part>.<dependent> against the newest trunk
// entry's fingerprint, A-78 a). Kind 6: the delivered version's catalog entry no longer serves the verb or word (A-78 b). A
// stale edge (kinds 1, 3) has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that
// left the digest as it was makes nothing behind.
func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (meshcatalog.OutdatedAnswer, error) {
out := meshcatalog.OutdatedAnswer{Behind: []meshcatalog.Outdated{}}
if ix.Versions == nil || ix.Heard == nil {
return out, errors.New("the state versions is not kept here")
}
all, err := ix.Versions.Keys(ctx)
if err != nil {
return out, err
}
modules := map[string]bool{}
for _, k := range all {
if m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k); ok {
modules[m] = true
}
}
newest := map[string]meshcatalog.VersionEntry{}
for m := range modules {
vs, err := ModuleVersions(ctx, ix.Versions, m)
if err != nil {
return out, err
}
for _, v := range vs {
if !v.Late {
newest[m] = v
break
}
}
}
trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them
if ix.Revisions != nil {
if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found {
if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil {
for _, e := range entries {
trunk[e.Module] = e
}
}
}
}
for _, m := range sortedKeys(newest) {
v := newest[m]
for _, e := range v.Edges {
dep := strings.SplitN(e.Part, ":", 2)[0]
if in.Dependency != "" && in.Dependency != e.Part {
continue
}
o := meshcatalog.Outdated{Module: m, Version: v.Version, Kind: e.Kind,
Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: ix.firstBranch(ctx, dep)}}
switch e.Kind {
case 1:
n, ok := newest[dep]
if !ok {
continue
}
var digest string
for _, a := range n.Artifacts {
if a.Artifact == e.Part {
digest = a.Digest
}
}
if digest == "" || digest == e.Digest {
continue
}
o.Recorded, o.Newest = e.Version+" "+e.Digest, n.Version+" "+digest
case 3:
tc, ok := ix.agreedToolchain(ctx, languageOf(trunk[m], e))
if !ok || tc.Version == "" || tc.Version == e.Version {
continue
}
o.Recorded, o.Newest = e.Version+" "+e.Digest, tc.Version
case 2, 4:
d, ok := ix.deliveredOf(ctx, dep)
if !ok || e.Version == notYetDelivered {
continue
}
digest := d.Parts[e.Part]
if digest == "" || digest == e.Digest {
continue
}
o.Recorded, o.Newest = e.Version+" "+e.Digest, d.Version+" "+digest
case 5:
cur := trunk[m].Fp
if cur == "" {
continue
}
acted, ok := ix.actedFor(ctx, e.Part, m)
if ok && acted.Build == cur {
continue
}
o.Recorded, o.Newest = "acted for "+cmp.Or(acted.Build, "no build"), "the current build "+cur
case 6:
d, ok := ix.deliveredOf(ctx, dep)
if !ok || e.Version == notYetDelivered || d.Version == e.Version {
continue
}
if serves, known := ix.deliveredServes(ctx, dep, d.Version, e.Word); !known || serves {
continue
}
o.Recorded, o.Newest = e.Version+" "+e.Word, d.Version+" no longer serves "+e.Word
default:
continue
}
if e.Stale {
o.Act = "rebuild"
}
out.Behind = append(out.Behind, o)
}
}
behind, err := ix.outdatedBranchBuilds(ctx, in)
if err != nil {
return out, err
}
out.Behind = append(out.Behind, behind...)
return out, nil
}
// agreedToolchain is the newest toolchain-agreed kept for a language (A-80).
func (ix *Indexer) agreedToolchain(ctx context.Context, language string) (meshbuilds.ToolchainAgreed, bool) {
var t meshbuilds.ToolchainAgreed
if language == "" {
return t, false
}
raw, _, found, err := ix.Heard.Get(ctx, ToolchainPrefix+keys.EscapeName(language))
if err != nil || !found || json.Unmarshal(raw, &t) != nil {
return t, false
}
return t, true
}
// languageOf is the language of the artifact a kind-3 edge of the entry builds: the target of the catalog graph's edge from the
// toolchain part, else the one language the entry's artifacts share.
func languageOf(entry meshcatalog.CatalogEntry, e meshcatalog.VersionEdge) string {
lang := map[string]string{}
for _, a := range entry.Artifacts {
lang[a.Ref] = a.Language
}
for _, g := range entry.Edges {
if g.Kind == 3 && (g.Part == e.Part || g.From == e.Part) && lang[g.To] != "" {
return lang[g.To]
}
}
one := ""
for _, l := range lang {
if l == "" || (one != "" && one != l) {
return ""
}
one = l
}
return one
}
// deliveredServes says whether the catalog entry of a delivered version of module serves word, its verb (seat.verb, as the entry's
// serves lists it, the form `seat:` allowed) or a word of its manifest (A-78 b); known is false where that version's entry cannot be read.
func (ix *Indexer) deliveredServes(ctx context.Context, module, version, word string) (serves, known bool) {
if ix.Revisions == nil {
return false, false
}
vs, err := ModuleVersions(ctx, ix.Versions, module)
if err != nil {
return false, false
}
var source int64
for _, v := range vs {
if v.Version == version {
source = v.Source
}
}
if source == 0 {
return false, false
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return false, false
}
var numbers []int64
for _, n := range names {
if r, ok := TrunkRevisionOf(n); ok {
numbers = append(numbers, r)
}
}
slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) })
for _, n := range numbers { // the newest revision taken from the source revision the version was built at
rev, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", n))
if err != nil || rev.Source != source {
continue
}
for _, e := range entries {
if e.Module != module {
continue
}
w := strings.TrimPrefix(word, "seat:")
_, inManifest := e.Manifest[word]
return slices.Contains(e.Serves, w) || slices.Contains(e.Provides, w) || slices.Contains(e.Streams, w) || inManifest, true
}
}
return false, false
}
// firstBranch is the first watched branch of a module's registration, which its released versions are of.
func (ix *Indexer) firstBranch(ctx context.Context, module string) string {
if ix.Registry != nil {
if r, found, err := ix.Registry.Registration(ctx, module); err == nil && found && len(r.Branches) > 0 {
return r.Branches[0]
}
}
return ""
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
slices.Sort(out)
return out
}
@@ -0,0 +1,207 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The first fill of the versions (design 47 G §372): at the first start under G the state versions is filled once, as
// released, for each module from the highest published version over every artifact of the module that the builder's state
// versions held, read by the builder's tool builder_versions as a kept call; then versions-filled is said once, and the
// builder reserves nothing before it (§369).
const (
// BuilderVersionsCall is the tool the manifest names in its invokes (until that tool goes).
BuilderVersionsCall = "mesh-delivery-builder.builder_versions"
// VersionsFilledKey is the key of the state heard that says the fill is done, so it is not made again.
VersionsFilledKey = "versions-filled"
)
// versionsFillSince is the key of the state heard of the time builder_versions is first unanswered.
var versionsFillSince = "call-since." + keys.EscapeName(BuilderVersionsCall)
// BuilderVersion is one version of a name in the answer of builder_versions (its fields the fill reads).
type BuilderVersion struct {
Version string `json:"version"`
State string `json:"state"`
Digest string `json:"digest"`
Revision string `json:"revision"`
At time.Time `json:"at"`
}
// BuilderVersionsAnswer is what builder_versions answers for a name.
type BuilderVersionsAnswer struct {
Name string `json:"name"`
Versions []BuilderVersion `json:"versions"`
}
// VersionsAsker asks builder_versions for a name's versions (a kept call, ADR 0668 §1, §2).
type VersionsAsker func(ctx context.Context, name string) (BuilderVersionsAnswer, error)
// versionNameOf is the name the builder numbers an artifact under: its package's name in its ecosystem, else
// <module>/<artifact> (images, bundles and archives).
func versionNameOf(module string, a meshcatalog.Artifact) string {
if a.Package != nil && a.Package.Name != "" {
return a.Package.Name
}
return module + "/" + artifactName(a.Ref)
}
// FillVersions fills the state versions once from builder_versions and says versions-filled (§372). A fill already made
// is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding
// call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if !ix.versionsFilling.CompareAndSwap(false, true) {
return nil // one fill runs at a time
}
defer ix.versionsFilling.Store(false)
if _, _, done, err := ix.Heard.Get(ctx, VersionsFilledKey); err != nil || done {
return err
}
if ix.Revisions == nil {
return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
return err
}
if !found {
return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return err
}
type found1 struct {
v BuilderVersion
name string
ref string
}
best := map[string][]found1{} // by module: each artifact's highest published version
var unanswered error
for _, e := range entries {
var top string // the module's highest published version
per := []found1{}
for _, a := range e.Artifacts {
name := versionNameOf(e.Module, a)
ans, err := ask(ctx, name)
if err != nil {
unanswered = err
continue
}
for _, v := range ans.Versions {
if _, ok := versionParts(v.Version); !ok || v.State != "published" {
continue
}
per = append(per, found1{v: v, name: name, ref: a.Ref})
if top == "" || compareVersions(v.Version, top) > 0 {
top = v.Version
}
}
}
for _, f := range per {
if f.v.Version == top {
best[e.Module] = append(best[e.Module], f)
}
}
}
if unanswered != nil {
ix.fillUnanswered(ctx, unanswered)
return fmt.Errorf("builder_versions is not answered: %w", unanswered)
}
ix.answered(ctx, versionsFillSince)
if ix.Findings != nil {
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, "builder_versions answers"); err != nil {
ix.logf("versions fill: the finding %s is not cleared: %v", BuilderVersionsCall, err)
}
}
now := ix.now().UTC()
filled := 0
built := map[string]meshcatalog.VersionEntry{}
claimed := map[string]time.Time{}
trunk := map[string]meshcatalog.CatalogEntry{}
for _, e := range entries {
trunk[e.Module] = e
}
for module, fs := range best {
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: fs[0].v.Version, At: &now}
for _, f := range fs {
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: f.ref, Digest: f.v.Digest})
if n, err := strconv.ParseInt(f.v.Revision, 10, 64); err == nil && n > v.Source {
v.Source = n
}
if f.v.At.After(claimed[module]) {
claimed[module] = f.v.At
}
}
built[module] = v
}
// The edge entries (§377), each marked filled: where the stream running or the placement cannot be read the entries of kinds 2, 4
// and 6 read "not yet delivered", as at a release, and the first delivery writes them.
var running map[string]map[string]nodeengine.Running
if ix.RunningWhole != nil {
if running, err = ix.RunningWhole(ctx); err != nil {
ix.logf("versions fill: the stream running is not read whole: %v", err)
running = nil
}
}
answers := map[string]BuilderVersionsAnswer{}
for _, module := range slices.Sorted(maps.Keys(built)) {
v := built[module]
v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, running, answers)
built[module] = v
}
for _, module := range slices.Sorted(maps.Keys(built)) {
v := built[module]
data, err := json.Marshal(v)
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil && !errors.Is(err, ErrConflict) {
return fmt.Errorf("the version %s %s is not kept: %w", module, v.Version, err)
}
ix.keepFilledFacts(ctx, v, trunk[module], built, now)
filled++
}
if ix.SayVersionsFilled != nil {
ev := meshcatalog.VersionsFilledEvent{V: int64(keys.Version), At: now, Modules: int64(filled)}
if err := ix.SayVersionsFilled(ctx, ev, meshcatalog.EventVersionsFilled); err != nil {
return fmt.Errorf("versions-filled is not said: %w", err)
}
}
if err := ix.Heard.Create(ctx, VersionsFilledKey, []byte(now.Format(time.RFC3339Nano))); err != nil && !errors.Is(err, ErrConflict) {
return err
}
ix.logf("versions: filled for %d modules from builder_versions, versions-filled said", filled)
return nil
}
// fillUnanswered writes call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
func (ix *Indexer) fillUnanswered(ctx context.Context, cause error) {
now := ix.now()
since := ix.unansweredSince(ctx, versionsFillSince, now)
if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter {
return
}
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered,
fmt.Sprintf("the call %s is unanswered since %s", BuilderVersionsCall, since.UTC().Format(time.RFC3339)),
"A call whose answer is kept is not answered",
fmt.Sprintf("The first fill of the catalog's versions asks %s of every artifact; it is not answered since %s (%v), so no version is released and the builder reserves nothing.", BuilderVersionsCall, since.UTC().Format(time.RFC3339), cause),
"indexer."+mcc.KindCallUnanswered)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("versions fill: the finding %s is not written: %v", BuilderVersionsCall, err)
}
}
@@ -0,0 +1,159 @@
package index
import (
"cmp"
"context"
"encoding/json"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The edge entries of the first fill (design 47 G §377, A-78 c): each module's newest version gets its entries once, each marked
// filled. Kinds 1 and 3 come from the dependency version published latest before the version's build was claimed, read with
// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 from the version every machine placing the dependency
// runs, present or away, the machines from the assigner's placement (A-62) and each node's last running from the node-engine seat's
// stream read whole (§556); kind 5 as done for every acting part's running build.
// PlacementAsker asks the seat mesh-module-assigner's verb placement for a module (a read, A-62).
type PlacementAsker func(ctx context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error)
// RunningReader reads the node-engine seat's stream running whole, every subject's last message (§556): by node, then module.
type RunningReader func(ctx context.Context) (map[string]map[string]nodeengine.Running, error)
// filledEdges are the edge entries of the entry's newest version, which was filled as built; claimed is when its build was claimed.
func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEntry, claimed time.Time, ask VersionsAsker, trunk map[string]meshcatalog.CatalogEntry,
built map[string]meshcatalog.VersionEntry, running map[string]map[string]nodeengine.Running, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge {
var out []meshcatalog.VersionEdge
seen := func(e meshcatalog.VersionEdge) bool {
return slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word })
}
for _, g := range entry.Edges {
part := cmp.Or(g.Part, g.From)
module := strings.SplitN(part, ":", 2)[0]
if part == "" || module == entry.Module {
continue
}
e := meshcatalog.VersionEdge{Kind: g.Kind, Part: part, Word: g.Word, Filled: true}
switch g.Kind {
case 1, 3:
dep, ok := trunk[module]
if !ok {
continue
}
var name string
for _, a := range dep.Artifacts {
if a.Ref == part {
name = versionNameOf(module, a)
}
}
if name == "" {
continue
}
ans, ok := answers[name]
if !ok {
a, err := ask(ctx, name)
if err != nil {
continue
}
ans, answers[name] = a, a
}
var best *BuilderVersion
for i, v := range ans.Versions {
if _, ok := versionParts(v.Version); !ok || v.State != "published" || v.At.After(claimed) {
continue
}
if best == nil || v.At.After(best.At) {
best = &ans.Versions[i]
}
}
if best == nil {
continue
}
e.Version, e.Digest, e.Stale = best.Version, best.Digest, true
case 2, 4, 6:
e.Version = notYetDelivered
if v, ok := ix.runningVersion(ctx, module, trunk, built, running); ok {
e.Version = v.Version
for _, a := range v.Artifacts {
if a.Artifact == part {
e.Digest = a.Digest
}
}
}
case 5:
e.Fact = "acted for the running build " + entry.Fp
default:
continue
}
if !seen(e) {
out = append(out, e)
}
}
return out
}
// runningVersion is the version of a module that every machine placing it runs, present or away, when each runs the newest
// build; false where no machine places it, a placed machine runs none or another build, or the placement is not answered.
func (ix *Indexer) runningVersion(ctx context.Context, module string, trunk map[string]meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry,
running map[string]map[string]nodeengine.Running) (meshcatalog.VersionEntry, bool) {
v, ok := built[module]
e, known := trunk[module]
if !ok || !known || e.Fp == "" || ix.Placement == nil {
return v, false
}
placements, err := ix.Placement(ctx, module)
if err != nil {
ix.logf("versions fill: the placement of %s is not answered: %v", module, err)
return v, false
}
nodes := 0
for _, p := range placements {
for _, n := range p.Nodes {
nodes++
r, ok := running[n][module]
if !ok || r.Stopped || r.Fingerprint != e.Fp {
return v, false
}
}
}
return v, nodes > 0
}
// keepFilledFacts keeps what the first fill derived of a version as heard: for each entry of kinds 2, 4 and 6 read as delivered,
// the dependency's version as its newest delivered one (A-68), and for each kind-5 edge the act for the running build (A-78 a).
func (ix *Indexer) keepFilledFacts(ctx context.Context, v meshcatalog.VersionEntry, entry meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, at time.Time) {
for _, e := range v.Edges {
switch e.Kind {
case 2, 4, 6:
dep := built[strings.SplitN(e.Part, ":", 2)[0]]
if e.Version == notYetDelivered || dep.Version != e.Version {
continue
}
parts := map[string]string{}
for _, a := range dep.Artifacts {
parts[a.Artifact] = a.Digest
}
d := meshwants.VersionDeliveredEvent{V: 1, Module: dep.Module, Version: dep.Version, Parts: parts, At: &at}
if err := ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(dep.Module), d, func([]byte) bool { return false }); err != nil {
ix.logf("versions fill: the delivered version of %s is not kept: %v", dep.Module, err)
}
case 5:
data, _ := json.Marshal(Acted{V: 1, Part: e.Part, Build: entry.Fp, How: "filled", At: at})
if err := write(ctx, ix.Heard, ActedKey(e.Part, v.Module), func(_ []byte, found bool) ([]byte, bool, error) {
if found {
return nil, false, nil
}
return data, false, nil
}); err != nil {
ix.logf("versions fill: the act for %s is not kept: %v", v.Module, err)
}
}
}
}
@@ -0,0 +1,532 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
)
// Design 47 G §349: a version is released once every artifact the module's entry names is published at it, said once as
// version-released; a version lower than the highest released is recorded late and not said; each module keeps its newest
// 100 versions, never its highest of any MAJOR.MINOR.
func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
var said []meshcatalog.VersionReleasedEvent
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now },
SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error {
said = append(said, e)
return nil
}}
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Manifest: map[string]any{},
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program"}, {Ref: "avahi:image"}}}}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
pub := func(artifact, version string) {
t.Helper()
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "avahi", Artifact: artifact, Digest: "sha256:" + artifact + version, Version: version, Source: 7, Branch: "main"}); err != nil {
t.Fatal(err)
}
}
pub("avahi:program", "0.1.5")
if v, _ := ModuleVersions(ctx, ix.Versions, "avahi"); len(v) != 0 || len(said) != 0 {
t.Fatalf("one of two artifacts published released %v, said %v", v, said)
}
pub("avahi:image", "0.1.5")
pub("avahi:image", "0.1.5") // heard again
v, _ := ModuleVersions(ctx, ix.Versions, "avahi")
if len(v) != 1 || len(v[0].Artifacts) != 2 || v[0].Source != 7 || v[0].Late || len(said) != 1 || said[0].Version != "0.1.5" {
t.Fatalf("released %+v, said %+v", v, said)
}
if keys, _ := ix.Heard.Keys(ctx); len(keys) != 0 {
t.Errorf("the artifacts kept as published stay after the release: %v", keys)
}
// a late older version is recorded and not said
pub("avahi:program", "0.1.2")
pub("avahi:image", "0.1.2")
v, _ = ModuleVersions(ctx, ix.Versions, "avahi")
if len(v) != 2 || v[1].Version != "0.1.2" || !v[1].Late || len(said) != 1 {
t.Fatalf("the late version: %+v, said %d", v, len(said))
}
// the newest 100 stay, and the highest of each MAJOR.MINOR
for i := 6; i < 6+VersionsKept+5; i++ {
pub("avahi:program", fmt.Sprintf("0.1.%d", i))
pub("avahi:image", fmt.Sprintf("0.1.%d", i))
}
for _, mm := range []string{"0.2", "0.3"} {
pub("avahi:program", mm+".0")
pub("avahi:image", mm+".0")
}
v, _ = ModuleVersions(ctx, ix.Versions, "avahi")
if len(v) != VersionsKept || v[0].Version != "0.3.0" {
t.Fatalf("%d versions kept, newest %s", len(v), v[0].Version)
}
var oldest string
for _, e := range v {
oldest = e.Version
}
if compareVersions(oldest, "0.1.5") < 0 {
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
}
}
// Design 47 G §372: the first fill records, once, each module's highest published version over its artifacts as released
// and says versions-filled once; a builder_versions that does not answer holds it and, after five minutes, is the finding
// call-unanswered; a fill made is not made again.
func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
w := newFindings()
var filledSaid []meshcatalog.VersionsFilledEvent
ix := &Indexer{Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now },
SayVersionsFilled: func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error {
filledSaid = append(filledSaid, e)
return nil
}}
entries := map[string]meshcatalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{
{Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}},
{Ref: "mesh-sdk:ts", Package: &meshcatalog.ArtifactPackage{Ecosystem: "npm", Name: "@novox/mesh-sdk"}}}},
"avahi": {Module: "avahi", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:image", Kind: "image"}}},
"fresh": {Module: "fresh", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "fresh:image", Kind: "image"}}},
}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
up := false
ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) {
if !up {
return BuilderVersionsAnswer{}, errors.New("no responders")
}
pub := func(v, d string) BuilderVersion {
return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9"}
}
switch name {
case "git.novox.be/novox/mesh-sdk/go":
return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.1.14", "sha256:go14"), {Version: "0.1.15", State: "refused"}}}, nil
case "@novox/mesh-sdk":
return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.1.18", "sha256:ts18")}}, nil
case "avahi/image":
return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.2.3", "sha256:av")}}, nil
}
return BuilderVersionsAnswer{Name: name}, nil
}
if err := ix.FillVersions(ctx, ask); err == nil || len(filledSaid) != 0 {
t.Fatalf("a builder_versions unanswered filled: %v", err)
}
if len(w.Held()) != 0 {
t.Fatal("a call unanswered for no time wrote its finding")
}
now = now.Add(5 * time.Minute)
_ = ix.FillVersions(ctx, ask)
if h := w.Held(); len(h) != 1 || h[0].ID != BuilderVersionsCall {
t.Fatalf("the findings after five minutes: %+v", h)
}
up = true
if err := ix.FillVersions(ctx, ask); err != nil {
t.Fatal(err)
}
if len(w.Held()) != 0 {
t.Errorf("the finding stands after builder_versions answered: %+v", w.Held())
}
if len(filledSaid) != 1 || filledSaid[0].Modules != 2 {
t.Fatalf("versions-filled said %+v", filledSaid)
}
sdk, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk")
if len(sdk) != 1 || sdk[0].Version != "0.1.18" || len(sdk[0].Artifacts) != 1 || sdk[0].Artifacts[0].Artifact != "mesh-sdk:ts" || sdk[0].Source != 9 {
t.Errorf("mesh-sdk's filled version: %+v (the higher of Go's 0.1.14 and npm's 0.1.18)", sdk)
}
if fresh, _ := ModuleVersions(ctx, ix.Versions, "fresh"); len(fresh) != 0 {
t.Errorf("a module with none published is filled: %+v", fresh)
}
if err := ix.FillVersions(ctx, ask); err != nil || len(filledSaid) != 1 {
t.Errorf("filled again: %v, said %d", err, len(filledSaid))
}
}
// Design 47 G §374, §375, A-68: a release records, per edge, the part it was built against (kind 1, from package-published's
// dependencies) or, once delivered, the delivered version it was ordered after (kinds 2 and 4); outdated answers an edge behind
// by the part's digest alone, with the act rebuild only for a stale edge; a newer version that left the digest as it was
// makes nothing behind.
func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
entries := map[string]meshcatalog.CatalogEntry{
"lib": {Module: "lib", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}},
"dep": {Module: "dep", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "dep:program"}}},
"app": {Module: "app", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}},
Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"},
{Kind: 4, From: "dep:program", To: "app:program", Part: "dep:program", Word: "provision"}}},
}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
pub := func(module, version, digest string, deps ...meshbuilds.Dependency) {
t.Helper()
if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: module, Artifact: module + ":" + map[string]string{"lib": "pkg", "dep": "program", "app": "program"}[module], Digest: digest, Version: version, Dependencies: deps}); err != nil {
t.Fatal(err)
}
}
behind := func() []meshcatalog.Outdated {
t.Helper()
a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{})
if err != nil {
t.Fatal(err)
}
return a.Behind
}
pub("lib", "0.1.1", "d1")
pub("dep", "0.1.1", "x1")
pub("app", "0.1.1", "a1", meshbuilds.Dependency{Kind: 1, Part: "lib:pkg", Version: "0.1.1", Digest: "d1"})
v, _ := ModuleVersions(ctx, ix.Versions, "app")
if len(v[0].Edges) != 2 || v[0].Edges[0].Kind != 1 || !v[0].Edges[0].Stale || v[0].Edges[0].Word != "imports" || v[0].Edges[1].Version != "not yet delivered" {
t.Fatalf("the release's edges: %+v", v[0].Edges)
}
pub("lib", "0.1.2", "d1") // a newer version, the part's digest as it was
if b := behind(); len(b) != 0 {
t.Fatalf("a newer version that left the digest as it was is behind: %+v", b)
}
pub("lib", "0.1.3", "d3")
if b := behind(); len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Act != "rebuild" || b[0].Dependency.Artifact != "lib:pkg" || !strings.Contains(b[0].Newest, "d3") || !strings.Contains(b[0].Recorded, "d1") {
t.Fatalf("behind after lib's new digest: %+v", b)
}
// delivery: dep 0.1.1 is delivered, then app 0.1.1 writes its kind 4 entry; dep 0.1.2 delivered later is behind, an ordering edge
deliver := func(module, version string, parts map[string]string) {
t.Helper()
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: module, Version: version, Parts: parts}); err != nil {
t.Fatal(err)
}
}
deliver("dep", "0.1.1", map[string]string{"dep:program": "x1"})
deliver("app", "0.1.1", map[string]string{"app:program": "a1"})
v, _ = ModuleVersions(ctx, ix.Versions, "app")
if e := v[0].Edges[1]; e.Version != "0.1.1" || e.Digest != "x1" {
t.Fatalf("the kind 4 entry after the first delivery: %+v", e)
}
deliver("dep", "0.1.2", map[string]string{"dep:program": "x2"})
deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) // delivered again: written once, not changed
v, _ = ModuleVersions(ctx, ix.Versions, "app")
if e := v[0].Edges[1]; e.Version != "0.1.1" {
t.Errorf("an entry written at the first delivery changed: %+v", e)
}
b := behind()
if len(b) != 2 || b[1].Kind != 4 || b[1].Act != "" || b[1].Dependency.Artifact != "dep:program" {
t.Fatalf("behind after dep's second delivery: %+v", b)
}
// the newest toolchain-agreed per language is kept as heard
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil {
t.Fatal(err)
}
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
t.Fatal(err)
}
if raw, _, found, _ := ix.Heard.Get(ctx, ToolchainPrefix+"go"); !found || !strings.Contains(string(raw), "1.26") {
t.Errorf("toolchain.go holds %s", raw)
}
}
// Design 47 G §354: an input is kept under its write time and sha256, read back by the verb input with its digest checked;
// the keys older than a day among the 20 oldest are deleted at each new keep.
func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }}
d1, err := ix.KeepInput(ctx, "ask-1", []byte("first"))
if err != nil {
t.Fatal(err)
}
a, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1})
if err != nil || a.Digest != d1 || a.Input != "Zmlyc3Q=" {
t.Fatalf("the input read back: %+v, %v", a, err)
}
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: "sha256:none"}); err == nil {
t.Error("an input never kept is answered")
}
now = now.Add(23 * time.Hour)
if _, err := ix.KeepInput(ctx, "ask-2", []byte("second")); err != nil {
t.Fatal(err)
}
inputsOnly := func() []string {
var out []string
all, _ := ix.Inputs.Keys(ctx)
for _, k := range all {
if !strings.HasPrefix(k, InputsAskPrefix) {
out = append(out, k)
}
}
return out
}
if keys := inputsOnly(); len(keys) != 2 {
t.Fatalf("inside a day: %v", keys)
}
now = now.Add(2 * time.Hour)
d3, _ := ix.KeepInput(ctx, "ask-3", []byte("third"))
if keys := inputsOnly(); len(keys) != 2 {
t.Fatalf("after a day the first stays: %v", keys)
}
if all, _ := ix.Inputs.Keys(ctx); slices.Contains(all, InputsAskPrefix+"ask-1") {
t.Errorf("the name by ask of the swept input stays: %v", all)
}
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil {
t.Error("the input a day old is still answered")
}
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d3}); err != nil {
t.Error(err)
}
}
// Design 47 G A-80, §73: what the indexer heard lives in the state heard, each key part escaped as a name, so a dotted module
// name parses and nothing of it is written to the own-build state.
func TestWhatIsHeardIsKeptEscapedInTheStateHeard(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "a.b", Version: "1.2.3"}); err != nil {
t.Fatal(err)
}
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
t.Fatal(err)
}
if _, ok := ix.deliveredOf(ctx, "a.b"); !ok {
t.Error("the delivered version of a dotted module is not read back")
}
got, _ := ix.Heard.Keys(ctx)
slices.Sort(got)
if want := []string{"delivered.a=2eb", "toolchain.go"}; !slices.Equal(got, want) {
t.Errorf("the keys of heard are %v, want %v", got, want)
}
if own, _ := ix.OwnBuildState.Keys(ctx); len(own) != 0 {
t.Errorf("the own-build state holds %v", own)
}
}
// Design 47 G §375, A-78, A-80: outdated says a kind-3 edge behind when the agreed toolchain of its artifact's language names another
// version, a kind-5 edge when the acting part has not acted for the dependent's current build (a wait-ended that ended ran or
// released), and a kind-6 edge when the delivered version's entry no longer serves the word.
func TestOutdatedAnswersKinds3And5And6(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
keep := func(n, source int64, serves []string, fp string) {
t.Helper()
entries := map[string]meshcatalog.CatalogEntry{
"prov": {Module: "prov", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "prov:program"}}, Serves: serves},
"actor": {Module: "actor", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}},
"app": {Module: "app", Manifest: map[string]any{}, Fp: fp, Artifacts: []meshcatalog.Artifact{{Ref: "app:program", Language: "go"}},
Edges: []meshcatalog.Edge{{Kind: 3, From: "tc:program", To: "app:program", Part: "tc:program"},
{Kind: 5, From: "actor:program", To: "app:program", Part: "actor:program"},
{Kind: 6, From: "prov:program", To: "app:program", Part: "prov:program", Word: "seat:s.verb"}}},
}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: n, Source: source, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
}
keep(1, 10, []string{"s.verb"}, "fp1")
put := func(module string, v meshcatalog.VersionEntry) {
t.Helper()
v.Module, v.V = module, 1
data, _ := json.Marshal(v)
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil {
t.Fatal(err)
}
}
put("prov", meshcatalog.VersionEntry{Version: "0.1.1", Source: 10})
put("app", meshcatalog.VersionEntry{Version: "0.1.1", Edges: []meshcatalog.VersionEdge{
{Kind: 3, Part: "tc:program", Version: "1.25", Digest: "t1", Stale: true},
{Kind: 5, Part: "actor:program"},
{Kind: 6, Part: "prov:program", Word: "seat:s.verb", Version: "0.1.1"}}})
behind := func() map[int64]meshcatalog.Outdated {
t.Helper()
a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{})
if err != nil {
t.Fatal(err)
}
m := map[int64]meshcatalog.Outdated{}
for _, o := range a.Behind {
m[o.Kind] = o
}
return m
}
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil {
t.Fatal(err)
}
b := behind()
if _, ok := b[3]; ok {
t.Errorf("kind 3 behind at the agreed version: %+v", b[3])
}
if o, ok := b[5]; !ok || o.Act != "" {
t.Errorf("kind 5 is not behind before the acting part acted: %+v", b)
}
wait := func(how, step, build string, since time.Time) {
t.Helper()
if err := ix.HearWaitEnded(ctx, meshwants.WaitEndedEvent{Part: "actor:program", Module: "app", Step: step, How: how, Version: build, Since: since}); err != nil {
t.Fatal(err)
}
}
at := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
wait("superseded", "compose", "fp1", at)
wait("ran", "delivery", "fp1", at)
if _, ok := behind()[5]; !ok {
t.Error("a superseded wait or a delivery wait counted as an act")
}
wait("ran", "compose", "fp1", at)
if _, ok := behind()[5]; ok {
t.Error("kind 5 is behind after the act for the current build")
}
keep(2, 20, []string{"s.other"}, "fp2")
if o, ok := behind()[5]; !ok || !strings.Contains(o.Newest, "fp2") {
t.Errorf("kind 5 after the dependent's new build: %+v", o)
}
wait("released", "compose", "fp2", at.Add(time.Minute))
if _, ok := behind()[5]; ok {
t.Error("a released wait is not an act")
}
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
t.Fatal(err)
}
if o, ok := behind()[3]; !ok || o.Act != "rebuild" || o.Newest != "1.26" || !strings.Contains(o.Recorded, "1.25") {
t.Errorf("kind 3 after the agreed toolchain changed: %+v", o)
}
// kind 6: prov 0.1.2 built at source 20 serves another word only
if _, ok := behind()[6]; ok {
t.Error("kind 6 behind with no newer delivered version")
}
put("prov", meshcatalog.VersionEntry{Version: "0.1.2", Source: 20})
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "prov", Version: "0.1.2", Parts: map[string]string{"prov:program": "p2"}}); err != nil {
t.Fatal(err)
}
if o, ok := behind()[6]; !ok || o.Act != "" || !strings.Contains(o.Newest, "no longer serves") {
t.Errorf("kind 6 after a version that serves no more: %+v", o)
}
}
// Design 47 G §377, A-78 c: the first fill gives each module's newest version its edge entries, each marked filled: kind 1 from the
// dependency version published latest before the build was claimed, kinds 2, 4 and 6 from the version every placing machine runs,
// read from the assigner's placement and the stream running, kind 5 as acted for the running build; so the first outdated says only what is truly behind.
func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) {
ctx := context.Background()
t0 := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
placed := map[string][]string{"prov": {"n1", "n2"}}
ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 },
Placement: func(_ context.Context, m string) ([]meshmoduleassigner.ModulePlacement, error) {
return []meshmoduleassigner.ModulePlacement{{Branch: "trunk", Nodes: placed[m]}}, nil
},
RunningWhole: func(context.Context) (map[string]map[string]nodeengine.Running, error) {
return map[string]map[string]nodeengine.Running{"n1": {"prov": {Fingerprint: "fp-prov"}}, "n2": {"prov": {Fingerprint: "fp-prov"}}}, nil
}}
entries := map[string]meshcatalog.CatalogEntry{
"lib": {Module: "lib", Manifest: map[string]any{}, Fp: "fp-lib", Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x/lib"}}}},
"actor": {Module: "actor", Manifest: map[string]any{}, Fp: "fp-actor", Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}},
"prov": {Module: "prov", Manifest: map[string]any{}, Fp: "fp-prov", Artifacts: []meshcatalog.Artifact{{Ref: "prov:program"}}},
"app": {Module: "app", Manifest: map[string]any{}, Fp: "fp-app", Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}},
Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true},
{Kind: 4, From: "prov:program", To: "app:program", Part: "prov:program"},
{Kind: 5, From: "actor:program", To: "app:program", Part: "actor:program"}}},
}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) {
pub := func(v, d string, at time.Time) BuilderVersion {
return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9", At: at}
}
switch name {
case "x/lib":
return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.1.1", "sha256:l1", t0), pub("0.1.2", "sha256:l2", t0.Add(2*time.Hour))}}, nil
case "app/program":
return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.3.1", "sha256:a1", t0.Add(time.Hour))}}, nil
}
return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.9.1", "sha256:"+name, t0)}}, nil
}
if err := ix.FillVersions(ctx, ask); err != nil {
t.Fatal(err)
}
app, _ := ModuleVersions(ctx, ix.Versions, "app")
if len(app) != 1 || len(app[0].Edges) != 3 {
t.Fatalf("app's filled version: %+v", app)
}
for _, e := range app[0].Edges {
if !e.Filled {
t.Errorf("a filled entry is not marked: %+v", e)
}
switch e.Kind {
case 1:
if e.Version != "0.1.1" || e.Digest != "sha256:l1" || !e.Stale {
t.Errorf("kind 1 is the version published before the build was claimed: %+v", e)
}
case 4:
if e.Version != "0.9.1" || e.Digest != "sha256:prov/program" {
t.Errorf("kind 4 is the version every machine runs: %+v", e)
}
}
}
// lib's 0.1.2 was published after app's build was claimed: that is behind, and nothing else (the delivered kind 4 and the act of kind 5 are not)
if a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{}); err != nil || len(a.Behind) != 1 || a.Behind[0].Kind != 1 {
t.Errorf("outdated after the first fill: %+v, %v", a.Behind, err)
}
}
// Design 47 G §354, A-78 d: an input is also keyed by its run ask's id, so the build seat's step-ended of that ask deletes it, and
// the key by ask no other read counts as an input.
func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }}
d1, err := ix.KeepInput(ctx, "a.1", []byte("first"))
if err != nil {
t.Fatal(err)
}
d2, _ := ix.KeepInput(ctx, "a.2", []byte("second"))
if _, err := ix.KeepInput(ctx, "", []byte("x")); err == nil {
t.Error("an input kept for no ask")
}
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "a.1"}); err != nil {
t.Fatal(err)
}
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "a.1"}); err != nil {
t.Errorf("an end heard again: %v", err)
}
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil {
t.Error("the input of an ended ask is still answered")
}
if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d2}); err != nil {
t.Errorf("the input of another ask: %v", err)
}
if got, _ := ix.Inputs.Keys(ctx); len(got) != 2 {
t.Errorf("keys after one end: %v", got)
}
if a, err := InputsReadTool(ix.Inputs).Run(map[string]any{}); err != nil || len(a.(InputsAnswer).Inputs) != 1 {
t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err)
}
}
@@ -130,7 +130,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
{Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
t.Errorf("state = %+v, want %+v", m.State, want)
}
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
@@ -164,9 +165,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
// internal/broker/state.go:62-93).
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments",
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments",
"mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
"mesh-delivery.source-moved", "mesh-wants.wanted", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
"mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -178,7 +179,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence"}; !reflect.DeepEqual(m.Invokes, want) {
if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
@@ -201,8 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool) // design 47 G §667, §8
if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || slices.Contains(m.Tools, "modules") {
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool) // design 47 G §667, §8
if !reflect.DeepEqual(m.Tools, want) || len(want) != 47 || slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
@@ -306,7 +307,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools
// bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own.
if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch"}) {
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions", "outdated", "input"}) {
t.Errorf("claims = %+v, want mesh-catalog with its seven verbs (entry: design 47 G §675)", m.Claims)
}
for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} {
+20 -4
View File
@@ -19,7 +19,10 @@
"register",
"policy",
"watch",
"unwatch"
"unwatch",
"versions",
"outdated",
"input"
]
}
],
@@ -69,7 +72,8 @@
"indexer_named_drop",
"indexer_proposals",
"indexer_proposal_drop",
"indexer_seed_again"
"indexer_seed_again",
"indexer_inputs"
],
"consumes": [
"go-package-registry.published",
@@ -80,6 +84,11 @@
"mesh-delivery-checker.preview.written",
"mesh-delivery-checker.try.ended",
"mesh-delivery-checker.try.started",
"mesh-builds.package-published",
"mesh-builds.toolchain-agreed",
"mesh-wants.version-delivered",
"node-build-go.step-ended",
"mesh-wants.wait-ended",
"mesh-delivery-checker.verdict",
"mesh-delivery-forge-bridge.forge.**",
"mesh-delivery.source-moved",
@@ -111,7 +120,10 @@
{ "name": "entry-index", "history": 1, "max-mib": 8 },
{ "name": "named", "history": 1, "max-mib": 4 },
{ "name": "registrations", "history": 1 },
{ "name": "proposals", "history": 1, "max-mib": 4 }
{ "name": "proposals", "history": 1, "max-mib": 4 },
{ "name": "versions", "history": 1, "max-mib": 64 },
{ "name": "build-edges", "history": 1 },
{ "name": "inputs", "history": 1 }
],
"streams": [
{ "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 },
@@ -155,7 +167,11 @@
"seat:mesh-module-assigner.assignments",
"seat:mesh-module-assigner.placement",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-watchdogs.presence"
"seat:mesh-watchdogs.presence",
"seat:node-build-go.steps",
"seat:node-build-node.steps",
"seat:node-build-image.steps",
"mesh-delivery-builder.builder_versions"
],
"uses": [
"operator-channel"