From c0e2621bba71470e3a3c9536251eaec941d8ea31 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:56:06 +0100 Subject: [PATCH 01/13] indexer: the seed asks steps of each artifact's build seat from the newest trunk entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seat list's build seat per artifact (language or kind), a kept call .steps with {module, artifact, fingerprint, fetch}, and the finding call-unanswered (scope call, id .steps) after five minutes, cleared on an answer. Builds: design 47 G §704 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer/main.go | 8 +- .../cmd/mesh-delivery-indexer/steps.go | 28 +++ .../internal/index/candidates.go | 1 + .../internal/index/seed_steps.go | 178 ++++++++++++++++++ .../internal/index/seed_steps_test.go | 93 +++++++++ .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 5 +- 7 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go create mode 100644 modules/mesh-delivery-indexer/internal/index/seed_steps.go create mode 100644 modules/mesh-delivery-indexer/internal/index/seed_steps_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 5beaf5bc..60bf0672 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -452,8 +452,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)}, routerRecordUnread, ix, logger.Printf) ix.Registry = registry - ix.SeedModules = func(context.Context) ([]index.SeedModule, error) { - return nil, errors.New("the first registrations are not composed by this build: what the build seats answer steps with is not read yet (design question)") + ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { + r := seatReader.Reader() + if r == nil { + return nil, errors.New("the seat list is not read yet: the seed waits for it") + } + return ix.SeedModulesFrom(ctx, r.Definitions(), stepsVia(calls.CallKept)) } // Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the // node-engine names the successor while it replaces this process, and an old process that learnt it later would take diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go new file mode 100644 index 00000000..4bd55f18 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + + "mesh-delivery-indexer/internal/index" +) + +// stepsVia asks a build seat's verb steps through call (process.Conn.CallKept, ADR 0668 §1) and reads its answer (design +// 47 G §704); the seats' verbs have one form, so nodebuildgo's contract reads each seat's answer. +func stepsVia(call toolCaller) index.StepsCaller { + return func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) { + verb := "seat:" + seat + "." + index.StepsVerb + raw, err := call(verb, in) + if err != nil { + return nodebuildgo.StepsAnswer{}, err + } + var a nodebuildgo.StepsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return a, fmt.Errorf("the answer of %s is not its steps: %w", verb, err) + } + return a, nil + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index d86201b2..b9726485 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -149,6 +149,7 @@ type Indexer struct { previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) + stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go new file mode 100644 index 00000000..ff6994e0 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -0,0 +1,178 @@ +package index + +import ( + "context" + "fmt" + "maps" + "path" + "slices" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// StepsVerb is the verb of each build seat the seed asks (design 47 G §704). +const StepsVerb = "steps" + +// StepsCaller asks the build seat seat its verb steps (a kept call, ADR 0668 §1): the answer, or why none came. +type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) + +// ModuleManifest is the file name of a module's manifest in its directory. +const ModuleManifest = "module.json" + +// buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704, +// §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said. +func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) { + var seats []string + for _, name := range slices.Sorted(maps.Keys(defs)) { + for _, r := range defs[name].Acts { + if r.Step != seatdef.StepBuild { + continue + } + if r.ActsOn(a.Language) || (r.On == seatdef.OnKindImage && a.Kind == "image") { + seats = append(seats, name) + break + } + } + } + switch len(seats) { + case 0: + return "", fmt.Errorf("no seat builds the artifact %s (language %q, kind %q)", a.Ref, a.Language, a.Kind) + case 1: + return seats[0], nil + } + return "", fmt.Errorf("the seats %v each build the artifact %s", seats, a.Ref) +} + +// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of +// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk +// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves +// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id .steps) is written once +// it is unanswered five minutes and cleared when it answers. +func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) { + if ix.Revisions == nil { + return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from") + } + rev, found, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first") + } + _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) + if err != nil { + return nil, err + } + seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made + for _, e := range entries { + for _, a := range e.Artifacts { + seat, err := buildSeatOf(defs, a) + if err != nil { + return nil, fmt.Errorf("%s: %w", e.Module, err) + } + seatOf[a.Ref] = seat + } + } + answered := map[string]bool{} + unanswered := map[string]error{} + var out []SeedModule + for _, e := range entries { + m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)} + for _, a := range e.Artifacts { + seat := seatOf[a.Ref] + name := artifactName(a.Ref) + sa := SeedArtifact{Name: name, Seat: seat} + in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, + Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name)}} + ans, err := steps(ctx, seat, in) + switch { + case err != nil: + unanswered[seat] = err + case ans.Error != "": + unanswered[seat] = fmt.Errorf("%s", ans.Error) + default: + sa.Answered = true + answered[seat] = true + for _, s := range ans.Steps { + sa.Steps = append(sa.Steps, s.Step) + } + } + m.Artifacts = append(m.Artifacts, sa) + } + out = append(out, m) + } + ix.stepsFindings(ctx, answered, unanswered) + return out, nil +} + +// artifactName is the artifact's name in an artifact reference :. +func artifactName(ref string) string { + for i := len(ref) - 1; i >= 0; i-- { + if ref[i] == ':' { + return ref[i+1:] + } + } + return ref +} + +// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered. +func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) { + if ix.Findings == nil { + return + } + ix.mu.Lock() + if ix.stepsSince == nil { + ix.stepsSince = map[string]time.Time{} + } + now := ix.now() + var raise []string + for seat := range unanswered { + if _, ok := ix.stepsSince[seat]; !ok { + ix.stepsSince[seat] = now + } + if now.Sub(ix.stepsSince[seat]) >= mcc.CallUnansweredAfter { + raise = append(raise, seat) + } + } + for seat := range answered { + if _, bad := unanswered[seat]; !bad { + delete(ix.stepsSince, seat) + } + } + since := maps.Clone(ix.stepsSince) + ix.mu.Unlock() + for _, seat := range slices.Sorted(slices.Values(raise)) { + id := seat + "." + StepsVerb + f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered, + fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)), + "A call whose answer is kept is not answered", + fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]), + "indexer."+mcc.KindCallUnanswered) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("seed: the finding %s is not written: %v", id, err) + } + } + for seat := range answered { + if _, bad := unanswered[seat]; bad { + continue + } + if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil { + ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err) + } + } +} + +// namedPaths are the named paths the entry keeps for the artifact, by its name or its reference. +func namedPaths(e meshcatalog.CatalogEntry, name string) []string { + if e.Inputs == nil { + return nil + } + if p, ok := e.Inputs.Paths[name]; ok { + return p + } + return e.Inputs.Paths[ArtifactRefOf(e.Module, name)] +} diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go new file mode 100644 index 00000000..849994c5 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -0,0 +1,93 @@ +package index + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// Design 47 G §704: the seed asks `steps` of the build seat whose `build` rule matches each artifact, with {module, +// artifact, fingerprint, fetch} from the newest trunk entry; a seat that does not answer holds the seed and, after five +// minutes, is the finding call-unanswered (scope call, id .steps), cleared when it answers. +func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + w := newFindings() + ix := &Indexer{Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} + fp := "src2:" + strings.Repeat("a", 64) + entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, + Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, + Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + defs := map[string]seatdef.Definition{ + "node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}}, + "node-build-image": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: seatdef.OnKindImage}}}, + } + // the image artifact is built by both rules: the language's and the kind's + if _, err := ix.SeedModulesFrom(ctx, defs, nil); err == nil || !strings.Contains(err.Error(), "each build") { + t.Fatalf("two seats building one artifact: %v", err) + } + e := entries["avahi"] + e.Artifacts[1].Language = "dockerfile" + entries["avahi"] = e + rev, objects, _ = CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 4, Kind: "trunk"}, entries) + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + var got []nodebuildgo.StepsInput + goUp := false + call := func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) { + got = append(got, in) + if seat == "node-build-go" && !goUp { + return nodebuildgo.StepsAnswer{}, errors.New("no responders") + } + return nodebuildgo.StepsAnswer{Steps: []nodebuildgo.DetectedStep{{Step: "unit"}}}, nil + } + mods, err := ix.SeedModulesFrom(ctx, defs, call) + if err != nil { + t.Fatal(err) + } + if len(mods) != 1 || mods[0].Module != "avahi" || mods[0].Origin != "novox/r" || mods[0].Path != "mods/avahi/module.json" || len(mods[0].Artifacts) != 2 { + t.Fatalf("the seed's modules: %+v", mods) + } + if a := mods[0].Artifacts[0]; a.Answered || a.Seat != "node-build-go" { + t.Errorf("the go artifact is %+v, want unanswered by node-build-go", a) + } + if a := mods[0].Artifacts[1]; !a.Answered || a.Seat != "node-build-image" || len(a.Steps) != 1 { + t.Errorf("the image artifact is %+v", a) + } + in := got[0] + if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 { + t.Errorf("the steps input is %+v", in) + } + if len(w.Held()) != 0 { + t.Fatal("a call unanswered for no time wrote its finding") + } + now = now.Add(5 * time.Minute) + if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil { + t.Fatal(err) + } + if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != mcc.KindCallUnanswered { + t.Fatalf("the findings after five minutes: %+v", h) + } + goUp = true + if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil { + t.Fatal(err) + } + if len(w.Held()) != 0 { + t.Errorf("the finding stands after the seat answered: %+v", w.Held()) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index ce0067d4..027f6bf5 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -172,7 +172,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 450b07a0..96a3f32b 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -149,7 +149,10 @@ "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", - "seat:mesh-watchdogs.presence" + "seat:mesh-watchdogs.presence", + "seat:node-build-go.steps", + "seat:node-build-node.steps", + "seat:node-build-image.steps" ], "own-secrets": { "broker": "${dir:state}/broker" From 940889c2a2cb173004c4299ec7ddacd92c457af8 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:58:21 +0100 Subject: [PATCH 02/13] indexer: the seed's steps fetch carries the entry's packages; the time a seat is unanswered is kept in the own-build state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §704, A-66 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/index/seed_steps.go | 64 +++++++++++++++---- .../internal/index/seed_steps_test.go | 6 +- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index ff6994e0..587261ae 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -87,7 +87,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. name := artifactName(a.Ref) sa := SeedArtifact{Name: name, Seat: seat} in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, - Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name)}} + Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name), Packages: maps.Clone(e.Packages)}} ans, err := steps(ctx, seat, in) switch { case err != nil: @@ -124,27 +124,20 @@ func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, if ix.Findings == nil { return } - ix.mu.Lock() - if ix.stepsSince == nil { - ix.stepsSince = map[string]time.Time{} - } now := ix.now() var raise []string + since := map[string]time.Time{} for seat := range unanswered { - if _, ok := ix.stepsSince[seat]; !ok { - ix.stepsSince[seat] = now - } - if now.Sub(ix.stepsSince[seat]) >= mcc.CallUnansweredAfter { + since[seat] = ix.stepsUnansweredSince(ctx, seat, now) + if now.Sub(since[seat]) >= mcc.CallUnansweredAfter { raise = append(raise, seat) } } for seat := range answered { if _, bad := unanswered[seat]; !bad { - delete(ix.stepsSince, seat) + ix.stepsAnswered(ctx, seat) } } - since := maps.Clone(ix.stepsSince) - ix.mu.Unlock() for _, seat := range slices.Sorted(slices.Values(raise)) { id := seat + "." + StepsVerb f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered, @@ -176,3 +169,50 @@ func namedPaths(e meshcatalog.CatalogEntry, name string) []string { } return e.Inputs.Paths[ArtifactRefOf(e.Module, name)] } + +// StepsSincePrefix opens the keys of the own-build state (OwnBuildState) that keep, by build seat, since when its steps +// has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past +// five minutes. +const StepsSincePrefix = "steps-since." + +// stepsUnansweredSince is the time the seat first failed to answer, kept in the indexer's own state; the first failure +// writes now. Where no state is kept it is held in memory. +func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time { + if ix.OwnBuildState == nil { + ix.mu.Lock() + defer ix.mu.Unlock() + if ix.stepsSince == nil { + ix.stepsSince = map[string]time.Time{} + } + if _, ok := ix.stepsSince[seat]; !ok { + ix.stepsSince[seat] = now + } + return ix.stepsSince[seat] + } + key := StepsSincePrefix + seat + if v, _, found, err := ix.OwnBuildState.Get(ctx, key); err == nil && found { + if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil { + return t + } + } + if err := ix.OwnBuildState.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil { + ix.logf("seed: the time since %s.%s is unanswered is not kept: %v", seat, StepsVerb, err) + } + return now +} + +// stepsAnswered forgets the time a seat that answers was unanswered since. +func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) { + if ix.OwnBuildState == nil { + ix.mu.Lock() + delete(ix.stepsSince, seat) + ix.mu.Unlock() + return + } + key := StepsSincePrefix + seat + if _, rev, found, err := ix.OwnBuildState.Get(ctx, key); err == nil && found { + if err := ix.OwnBuildState.Delete(ctx, key, rev); err != nil { + ix.logf("seed: the time since %s.%s is unanswered is not forgotten: %v", seat, StepsVerb, err) + } + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index 849994c5..f6c14b0e 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -20,10 +20,11 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { ctx := context.Background() now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) w := newFindings() - ix := &Indexer{Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} + ix := &Indexer{Findings: w, OwnBuildState: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} fp := "src2:" + strings.Repeat("a", 64) entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"}, Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}} rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) if err != nil { @@ -70,13 +71,14 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Errorf("the image artifact is %+v", a) } in := got[0] - if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 { + if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" { t.Errorf("the steps input is %+v", in) } if len(w.Held()) != 0 { t.Fatal("a call unanswered for no time wrote its finding") } now = now.Add(5 * time.Minute) + ix = &Indexer{Findings: w, OwnBuildState: ix.OwnBuildState, Revisions: ix.Revisions, Now: func() time.Time { return now }} // a restart keeps the time if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil { t.Fatal(err) } From e85432c9f656420a419ac5e1811ac3667479e333 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:02:12 +0100 Subject: [PATCH 03/13] indexer: hear package-published, release a version once every artifact is published, say version-released; the verb versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state versions (64 MiB), a late older version recorded and not said, the newest 100 kept per module and never the highest of a MAJOR.MINOR; the artifacts published so far are kept as pending.. in the own-build state. Builds: design 47 G §349, §369, A-69 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 7 + .../cmd/mesh-delivery-indexer/versions.go | 58 ++++ .../internal/index/candidates.go | 4 + .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/versions.go | 272 ++++++++++++++++++ .../internal/index/versions_test.go | 80 ++++++ .../mesh-delivery-indexer/manifest_test.go | 7 +- modules/mesh-delivery-indexer/module.json | 7 +- 10 files changed, 433 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go create mode 100644 modules/mesh-delivery-indexer/internal/index/versions.go create mode 100644 modules/mesh-delivery-indexer/internal/index/versions_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 23c30b89..b4bf550f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -288,7 +288,7 @@ func main() { tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)})) + tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)})) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 60bf0672..9795d4f5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -22,6 +22,7 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -441,6 +442,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). SayCatalogRevision: refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), SayFill: sayFill(conn, cred), + // The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released. + Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)}, + SayVersionReleased: sayVersionReleased(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, @@ -667,6 +671,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro _ = m.Ack() // nothing else the indexer consumes rides the shared stream } } + if err := hub.Handle(meshbuilds.SubjectPackagePublished, func(m jetstream.Msg) { heardPackagePublished(ctx, ix, m, logger.Printf) }); err != nil { + return err + } if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go new file mode 100644 index 00000000..a9822e1b --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "encoding/json" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-indexer/internal/index" +) + +// saySeatEvent publishes body as the seat mesh-catalog's event under the bus's duplicate id, as every event of the seat is +// said: published, never asked, and a refusal said on the connection's error handler. +func saySeatEvent(conn *nats.Conn, cred process.Account, event, id string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + msg := nats.NewMsg(seatEventSubject(event)) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + return conn.PublishMsg(msg) +} + +// sayVersionReleased says version-released (design 47 G §349) on mesh.seat.mesh-catalog.event.version-released. +func sayVersionReleased(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionReleasedEvent, string) error { + return func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error { + return saySeatEvent(conn, cred, meshcatalog.EventVersionReleased, id, e) + } +} + +// heardPackagePublished takes one package-published of the seat mesh-builds into the catalog's versions (design 47 G §349): +// one that does not decode is said and acknowledged; one that cannot be kept is left, so the bus gives it again. +func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var p meshbuilds.PackagePublished + if err := json.Unmarshal(m.Data(), &p); err != nil { + logf("%s holds what is no package-published, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.HearPackagePublished(ctx, p); err != nil { + logf("versions: the package-published of %s %s could not be kept, so the bus gives it again: %v", p.Artifact, p.Version, err) + _ = m.Nak() + return + } + _ = m.Ack() +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index b9726485..9c86e19f 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -107,6 +107,10 @@ type Indexer struct { // SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate // id given (W4.6). SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error + // Versions is the state versions (versions.go, design 47 G §349): each module's released versions; nil keeps none. + Versions Store + // SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing. + SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error // Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads // what the first registrations are composed from (§8). Nil: no registration act is answered. Registry *Registry diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 36a8b923..36f24c49 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -910,7 +910,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, OwnBuildState: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, + Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, // A dry run writes no registration and asks no one. Registry: nil, SeedModules: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index fc7481fe..58e4d7b3 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go new file mode 100644 index 00000000..8163a690 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -0,0 +1,272 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's +// released versions in the state versions, one key per module and version (.) holding each artifact's digest, +// the source revision built and its edge entries. It hears every artifact's package-published and records a version as +// released once every artifact the module's entry names is published at it. +const ( + VersionsState = "versions" + VersionsBucket = Module + "_" + VersionsState + // VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them + // go, never its highest of any MAJOR.MINOR. + VersionsKept = 100 + // PendingPrefix opens the keys of the own-build state (OwnBuildState) that keep, per module and version not yet + // released, the artifacts published at it so far (A-68's state, the one of A-66). + PendingPrefix = "pending." +) + +// published is what a not yet released version holds: each artifact published at it, by reference. +type published struct { + Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"` + Source int64 `json:"source,omitempty"` +} + +// versionParts is MAJOR.MINOR.PATCH, read; false for any other form. +func versionParts(v string) ([3]int, bool) { + var out [3]int + parts := strings.Split(v, ".") + if len(parts) != 3 { + return out, false + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n < 0 { + return out, false + } + out[i] = n + } + return out, true +} + +// compareVersions orders two versions MAJOR.MINOR.PATCH numerically. +func compareVersions(a, b string) int { + x, _ := versionParts(a) + y, _ := versionParts(b) + for i := range x { + if x[i] != y[i] { + return x[i] - y[i] + } + } + return 0 +} + +// ModuleVersions reads every released version the state keeps for module, newest first. +func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) { + all, err := st.Keys(ctx) + if err != nil { + return nil, err + } + var out []meshcatalog.VersionEntry + for _, k := range all { + m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k) + if !ok || m != module { + continue + } + raw, _, found, err := st.Get(ctx, k) + if err != nil { + return nil, err + } + if !found { + continue + } + v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw) + if err != nil { + return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err) + } + out = append(out, v) + } + slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) }) + return out, nil +} + +// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its +// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded, +// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A +// build of a watched branch other than the registration's first releases none (§371). +func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error { + if ix.Versions == nil || ix.OwnBuildState == nil { + return errors.New("the state versions is not kept here") + } + if _, ok := versionParts(p.Version); !ok { + return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version) + } + if ix.Registry != nil { + if r, found, err := ix.Registry.Registration(ctx, p.Module); err == nil && found && len(r.Branches) > 0 && r.Branches[0] != p.Branch { + return nil + } + } + if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil || found { + return err // released already: heard again + } + want, err := ix.artifactsOf(ctx, p.Module) + if err != nil { + return err + } + key := PendingPrefix + meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version) + var have published + err = write(ctx, ix.OwnBuildState, key, func(raw []byte, found bool) ([]byte, bool, error) { + have = published{} + if found { + if err := json.Unmarshal(raw, &have); err != nil { + return nil, false, err + } + } + if have.Artifacts == nil { + have.Artifacts = map[string]meshcatalog.VersionArtifact{} + } + have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest} + have.Source = p.Source + data, err := json.Marshal(have) + return data, false, err + }) + if err != nil { + return err + } + for _, ref := range want { + if _, ok := have.Artifacts[ref]; !ok { + return nil + } + } + return ix.releaseVersion(ctx, p, key, want, have) +} + +// artifactsOf are the artifact references the module's entry in the newest trunk revision names. +func (ix *Indexer) artifactsOf(ctx context.Context, module string) ([]string, error) { + if ix.Revisions == nil { + return nil, errors.New("the catalog graph is not kept here: no entry names the module's artifacts") + } + rev, found, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + return nil, err + } + if !found { + return nil, errors.New("the indexer keeps no trunk revision yet") + } + _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Module != module { + continue + } + var refs []string + for _, a := range e.Artifacts { + refs = append(refs, a.Ref) + } + return refs, nil + } + return nil, fmt.Errorf("the trunk holds no module %s", module) +} + +// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions, +// and forgets the artifacts kept as published. +func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, want []string, have published) error { + key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version) + kept, err := ModuleVersions(ctx, ix.Versions, p.Module) + if err != nil { + return err + } + late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0 + at := ix.now().UTC() + v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late} + for _, ref := range slices.Sorted(slices.Values(want)) { + v.Artifacts = append(v.Artifacts, have.Artifacts[ref]) + } + data, err := json.Marshal(v) + if err != nil { + return err + } + if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { + return err + } + if !late && ix.SayVersionReleased != nil { + ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at} + if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil { + return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err) + } + } + if err := ix.trimVersions(ctx, p.Module); err != nil { + ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err) + } + return ix.forgetPublished(ctx, pendingKey) +} + +// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR. +func (ix *Indexer) trimVersions(ctx context.Context, module string) error { + kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first + if err != nil || len(kept) <= VersionsKept { + return err + } + highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first + seen := map[string]bool{} + for _, v := range kept { + p := strings.Split(v.Version, ".") + if mm := p[0] + "." + p[1]; !seen[mm] { + seen[mm] = true + highest[v.Version] = true + } + } + excess := len(kept) - VersionsKept + for i := len(kept) - 1; i >= 0 && excess > 0; i-- { + v := kept[i].Version + if highest[v] { + continue + } + key := meshcatalog.VersionsByModuleVersionKey(module, v) + if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found { + if err := ix.Versions.Delete(ctx, key, rev); err != nil { + return err + } + } + excess-- + } + return nil +} + +// forgetPublished removes the artifacts kept as published at a version released. +func (ix *Indexer) forgetPublished(ctx context.Context, key string) error { + if _, rev, found, err := ix.OwnBuildState.Get(ctx, key); err != nil || !found { + return err + } else { + return ix.OwnBuildState.Delete(ctx, key, rev) + } +} + +// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first. +func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) { + if in.Module == "" { + return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)") + } + v, err := ModuleVersions(ctx, st, in.Module) + if v == nil { + v = []meshcatalog.VersionEntry{} + } + return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err +} + +// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime. +func VersionsTool(st Store) stdio.Tool { + return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) { + var in meshcatalog.VersionsInput + if raw, err := json.Marshal(args); err == nil { + _ = json.Unmarshal(raw, &in) + } + return Versions(context.Background(), st, in) + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go new file mode 100644 index 00000000..5fbf0fe0 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -0,0 +1,80 @@ +package index + +import ( + "context" + "fmt" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G §349: a version is released once every artifact the module's entry names is published at it, said once as +// version-released; a version lower than the highest released is recorded late and not said; each module keeps its newest +// 100 versions, never its highest of any MAJOR.MINOR. +func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + var said []meshcatalog.VersionReleasedEvent + ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, + SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error { + said = append(said, e) + return nil + }} + entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Manifest: map[string]any{}, + Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program"}, {Ref: "avahi:image"}}}} + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + pub := func(artifact, version string) { + t.Helper() + if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "avahi", Artifact: artifact, Digest: "sha256:" + artifact + version, Version: version, Source: 7, Branch: "main"}); err != nil { + t.Fatal(err) + } + } + pub("avahi:program", "0.1.5") + if v, _ := ModuleVersions(ctx, ix.Versions, "avahi"); len(v) != 0 || len(said) != 0 { + t.Fatalf("one of two artifacts published released %v, said %v", v, said) + } + pub("avahi:image", "0.1.5") + pub("avahi:image", "0.1.5") // heard again + v, _ := ModuleVersions(ctx, ix.Versions, "avahi") + if len(v) != 1 || len(v[0].Artifacts) != 2 || v[0].Source != 7 || v[0].Late || len(said) != 1 || said[0].Version != "0.1.5" { + t.Fatalf("released %+v, said %+v", v, said) + } + if keys, _ := ix.OwnBuildState.Keys(ctx); len(keys) != 0 { + t.Errorf("the artifacts kept as published stay after the release: %v", keys) + } + // a late older version is recorded and not said + pub("avahi:program", "0.1.2") + pub("avahi:image", "0.1.2") + v, _ = ModuleVersions(ctx, ix.Versions, "avahi") + if len(v) != 2 || v[1].Version != "0.1.2" || !v[1].Late || len(said) != 1 { + t.Fatalf("the late version: %+v, said %d", v, len(said)) + } + // the newest 100 stay, and the highest of each MAJOR.MINOR + for i := 6; i < 6+VersionsKept+5; i++ { + pub("avahi:program", fmt.Sprintf("0.1.%d", i)) + pub("avahi:image", fmt.Sprintf("0.1.%d", i)) + } + for _, mm := range []string{"0.2", "0.3"} { + pub("avahi:program", mm+".0") + pub("avahi:image", mm+".0") + } + v, _ = ModuleVersions(ctx, ix.Versions, "avahi") + if len(v) != VersionsKept || v[0].Version != "0.3.0" { + t.Fatalf("%d versions kept, newest %s", len(v), v[0].Version) + } + var oldest string + for _, e := range v { + oldest = e.Version + } + if compareVersions(oldest, "0.1.5") < 0 { + t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 027f6bf5..9b42127a 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -124,7 +124,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, // design 47 G §695 - {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 + {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, + {Name: index.VersionsState, History: 1, MaxMiB: 64}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, @@ -158,7 +159,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.source-moved", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) @@ -300,7 +301,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools // bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own. if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch"}) { + !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions"}) { t.Errorf("claims = %+v, want mesh-catalog with its seven verbs (entry: design 47 G §675)", m.Claims) } for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 96a3f32b..33863ef4 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -19,7 +19,8 @@ "register", "policy", "watch", - "unwatch" + "unwatch", + "versions" ] } ], @@ -79,6 +80,7 @@ "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", + "mesh-builds.package-published", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.source-moved", @@ -107,7 +109,8 @@ { "name": "unjudged", "history": 1 }, { "name": "entry-index", "history": 1, "max-mib": 8 }, { "name": "registrations", "history": 1 }, - { "name": "proposals", "history": 1, "max-mib": 4 } + { "name": "proposals", "history": 1, "max-mib": 4 }, + { "name": "versions", "history": 1, "max-mib": 64 } ], "streams": [ { "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 }, From e9d29977603dec15d5ed6fbaedfbf8d5724d9083 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:04:14 +0100 Subject: [PATCH 04/13] indexer: the first fill of the versions from builder_versions, then versions-filled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filled once, as released, with each module's highest published version; an unanswered builder_versions holds it and, after five minutes, is call-unanswered. The edge entries of the first fill (§377) wait for answers. Builds: design 47 G §372 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer/main.go | 9 +- .../cmd/mesh-delivery-indexer/versions.go | 52 ++++++ .../internal/index/candidates.go | 45 ++--- .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/seed_steps.go | 32 ++-- .../internal/index/versions_fill.go | 174 ++++++++++++++++++ .../internal/index/versions_test.go | 79 ++++++++ .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 3 +- 10 files changed, 361 insertions(+), 39 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/versions_fill.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 9795d4f5..f5827821 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -431,6 +431,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } }) readWhenBound(seatReader, seatDefs, marks) + var fillFollow atomic.Pointer[versionsFill] ix := &index.Indexer{Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred), Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, @@ -440,11 +441,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). - SayCatalogRevision: refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), + SayCatalogRevision: afterRevision(refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), &fillFollow), SayFill: sayFill(conn, cred), // The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released. Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)}, - SayVersionReleased: sayVersionReleased(conn, cred), + SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, @@ -456,6 +457,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)}, routerRecordUnread, ix, logger.Printf) ix.Registry = registry + // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. + fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKept), logf: logger.Printf} + fillFollow.Store(fill) + fill.kick() ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { r := seatReader.Reader() if r == nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index a9822e1b..f8507c48 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/json" + "fmt" + "sync/atomic" "time" "github.com/nats-io/nats.go" @@ -56,3 +58,53 @@ func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.M } _ = m.Ack() } + +// sayVersionsFilled says versions-filled (design 47 G §372) once, on the seat's stream that keeps it with no age limit. +func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionsFilledEvent, string) error { + return func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error { + return saySeatEvent(conn, cred, meshcatalog.EventVersionsFilled, id, e) + } +} + +// versionsVia asks the builder's tool builder_versions for one name as a kept call (ADR 0668 §1, §2) and reads its answer. +func versionsVia(call toolCaller) index.VersionsAsker { + return func(_ context.Context, name string) (index.BuilderVersionsAnswer, error) { + var a index.BuilderVersionsAnswer + raw, err := call(index.BuilderVersionsCall, map[string]any{"name": name}) + if err != nil { + return a, err + } + if err := json.Unmarshal(raw, &a); err != nil { + return a, fmt.Errorf("the answer of %s is not a name's versions: %w", index.BuilderVersionsCall, err) + } + return a, nil + } +} + +// versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each +// catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next. +type versionsFill struct { + ix *index.Indexer + ctx context.Context + ask index.VersionsAsker + logf func(string, ...any) +} + +func (f *versionsFill) kick() { + go func() { + if err := f.ix.FillVersions(f.ctx, f.ask); err != nil { + f.logf("versions: the first fill waits: %v", err) + } + }() +} + +// afterRevision runs the fill's kick after each catalog revision kept, once the fill is set. +func afterRevision(say func(context.Context, meshcatalog.CatalogRevisionEvent) error, fill *atomic.Pointer[versionsFill]) func(context.Context, meshcatalog.CatalogRevisionEvent) error { + return func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error { + err := say(ctx, e) + if f := fill.Load(); f != nil { + f.kick() + } + return err + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 9c86e19f..a683c83b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -111,6 +111,8 @@ type Indexer struct { Versions Store // SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing. SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error + // SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing. + SayVersionsFilled func(ctx context.Context, e meshcatalog.VersionsFilledEvent, id string) error // Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads // what the first registrations are composed from (§8). Nil: no registration act is answered. Registry *Registry @@ -148,27 +150,28 @@ type Indexer struct { // closed message the indexer missed heals at the next one (issue 1126); nil asks none. ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error) - mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach - source sourceNow - previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) - toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) - buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) - stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on - fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu - freshMu sync.Mutex - recompute sync.Mutex // one Recompute at a time - untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) - filling atomic.Bool // a FillTaken runs in the background (KickFill) - fillAgain atomic.Bool // a kick arrived while it ran - atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished) - saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head - failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes + mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach + source sourceNow + previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) + toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) + buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) + versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) + stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) + unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) + retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one + retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on + fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu + freshMu sync.Mutex + recompute sync.Mutex // one Recompute at a time + untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) + filling atomic.Bool // a FillTaken runs in the background (KickFill) + fillAgain atomic.Bool // a kick arrived while it ran + atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished) + saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head + failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. OwnBuild func(ctx context.Context) (string, error) underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 36f24c49..a18efefe 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -910,7 +910,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, OwnBuildState: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, + Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, // A dry run writes no registration and asks no one. Registry: nil, SeedModules: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 58e4d7b3..e51ba1dc 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 587261ae..1453433f 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -175,44 +175,52 @@ func namedPaths(e meshcatalog.CatalogEntry, name string) []string { // five minutes. const StepsSincePrefix = "steps-since." -// stepsUnansweredSince is the time the seat first failed to answer, kept in the indexer's own state; the first failure -// writes now. Where no state is kept it is held in memory. +// stepsUnansweredSince is the time the seat first failed to answer. func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time { + return ix.unansweredSince(ctx, StepsSincePrefix+seat, now) +} + +// stepsAnswered forgets the time a seat that answers was unanswered since. +func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) { + ix.answered(ctx, StepsSincePrefix+seat) +} + +// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's own state (the first +// failure writes now); where no state is kept it is held in memory. +func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time { if ix.OwnBuildState == nil { ix.mu.Lock() defer ix.mu.Unlock() if ix.stepsSince == nil { ix.stepsSince = map[string]time.Time{} } - if _, ok := ix.stepsSince[seat]; !ok { - ix.stepsSince[seat] = now + if _, ok := ix.stepsSince[key]; !ok { + ix.stepsSince[key] = now } - return ix.stepsSince[seat] + return ix.stepsSince[key] } - key := StepsSincePrefix + seat if v, _, found, err := ix.OwnBuildState.Get(ctx, key); err == nil && found { if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil { return t } } if err := ix.OwnBuildState.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil { - ix.logf("seed: the time since %s.%s is unanswered is not kept: %v", seat, StepsVerb, err) + ix.logf("the time since %s is unanswered is not kept: %v", key, err) } return now } -// stepsAnswered forgets the time a seat that answers was unanswered since. -func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) { +// answered forgets the time the call kept at key was unanswered since. +func (ix *Indexer) answered(ctx context.Context, key string) { if ix.OwnBuildState == nil { ix.mu.Lock() - delete(ix.stepsSince, seat) + delete(ix.stepsSince, key) ix.mu.Unlock() return } - key := StepsSincePrefix + seat if _, rev, found, err := ix.OwnBuildState.Get(ctx, key); err == nil && found { if err := ix.OwnBuildState.Delete(ctx, key, rev); err != nil { - ix.logf("seed: the time since %s.%s is unanswered is not forgotten: %v", seat, StepsVerb, err) + ix.logf("the time since %s is unanswered is not forgotten: %v", key, err) } } } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go new file mode 100644 index 00000000..b52537f4 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -0,0 +1,174 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The first fill of the versions (design 47 G §372): at the first start under G the state versions is filled once, as +// released, for each module from the highest published version over every artifact of the module that the builder's state +// versions held, read by the builder's tool builder_versions as a kept call; then versions-filled is said once, and the +// builder reserves nothing before it (§369). +const ( + // BuilderVersionsCall is the tool the manifest names in its invokes (until that tool goes). + BuilderVersionsCall = "mesh-delivery-builder.builder_versions" + // VersionsFilledKey is the key of the own-build state that says the fill is done, so it is not made again. + VersionsFilledKey = "versions-filled" + // versionsFillSince is the own-build key of the time builder_versions is first unanswered. + versionsFillSince = "call-since." + BuilderVersionsCall +) + +// BuilderVersion is one version of a name in the answer of builder_versions (its fields the fill reads). +type BuilderVersion struct { + Version string `json:"version"` + State string `json:"state"` + Digest string `json:"digest"` + Revision string `json:"revision"` + At time.Time `json:"at"` +} + +// BuilderVersionsAnswer is what builder_versions answers for a name. +type BuilderVersionsAnswer struct { + Name string `json:"name"` + Versions []BuilderVersion `json:"versions"` +} + +// VersionsAsker asks builder_versions for a name's versions (a kept call, ADR 0668 §1, §2). +type VersionsAsker func(ctx context.Context, name string) (BuilderVersionsAnswer, error) + +// versionNameOf is the name the builder numbers an artifact under: its package's name in its ecosystem, else +// / (images, bundles and archives). +func versionNameOf(module string, a meshcatalog.Artifact) string { + if a.Package != nil && a.Package.Name != "" { + return a.Package.Name + } + return module + "/" + artifactName(a.Ref) +} + +// FillVersions fills the state versions once from builder_versions and says versions-filled (§372). A fill already made +// is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding +// call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers. +func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { + if ix.Versions == nil || ix.OwnBuildState == nil { + return errors.New("the state versions is not kept here") + } + if !ix.versionsFilling.CompareAndSwap(false, true) { + return nil // one fill runs at a time + } + defer ix.versionsFilling.Store(false) + if _, _, done, err := ix.OwnBuildState.Get(ctx, VersionsFilledKey); err != nil || done { + return err + } + if ix.Revisions == nil { + return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts") + } + rev, found, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + return err + } + if !found { + return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first") + } + _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) + if err != nil { + return err + } + type found1 struct { + v BuilderVersion + name string + ref string + } + best := map[string][]found1{} // by module: each artifact's highest published version + var unanswered error + for _, e := range entries { + var top string // the module's highest published version + per := []found1{} + for _, a := range e.Artifacts { + name := versionNameOf(e.Module, a) + ans, err := ask(ctx, name) + if err != nil { + unanswered = err + continue + } + for _, v := range ans.Versions { + if _, ok := versionParts(v.Version); !ok || v.State != "published" { + continue + } + per = append(per, found1{v: v, name: name, ref: a.Ref}) + if top == "" || compareVersions(v.Version, top) > 0 { + top = v.Version + } + } + } + for _, f := range per { + if f.v.Version == top { + best[e.Module] = append(best[e.Module], f) + } + } + } + if unanswered != nil { + ix.fillUnanswered(ctx, unanswered) + return fmt.Errorf("builder_versions is not answered: %w", unanswered) + } + ix.answered(ctx, versionsFillSince) + if ix.Findings != nil { + if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, "builder_versions answers"); err != nil { + ix.logf("versions fill: the finding %s is not cleared: %v", BuilderVersionsCall, err) + } + } + now := ix.now().UTC() + filled := 0 + for module, fs := range best { + v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: fs[0].v.Version, At: &now} + for _, f := range fs { + v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: f.ref, Digest: f.v.Digest}) + if n, err := strconv.ParseInt(f.v.Revision, 10, 64); err == nil && n > v.Source { + v.Source = n + } + } + data, err := json.Marshal(v) + if err != nil { + return err + } + if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil && !errors.Is(err, ErrConflict) { + return fmt.Errorf("the version %s %s is not kept: %w", module, v.Version, err) + } + filled++ + } + if ix.SayVersionsFilled != nil { + ev := meshcatalog.VersionsFilledEvent{V: int64(keys.Version), At: now, Modules: int64(filled)} + if err := ix.SayVersionsFilled(ctx, ev, meshcatalog.EventVersionsFilled); err != nil { + return fmt.Errorf("versions-filled is not said: %w", err) + } + } + if err := ix.OwnBuildState.Create(ctx, VersionsFilledKey, []byte(now.Format(time.RFC3339Nano))); err != nil && !errors.Is(err, ErrConflict) { + return err + } + ix.logf("versions: filled for %d modules from builder_versions, versions-filled said", filled) + return nil +} + +// fillUnanswered writes call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state). +func (ix *Indexer) fillUnanswered(ctx context.Context, cause error) { + now := ix.now() + since := ix.unansweredSince(ctx, versionsFillSince, now) + if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter { + return + } + f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, + fmt.Sprintf("the call %s is unanswered since %s", BuilderVersionsCall, since.UTC().Format(time.RFC3339)), + "A call whose answer is kept is not answered", + fmt.Sprintf("The first fill of the catalog's versions asks %s of every artifact; it is not answered since %s (%v), so no version is released and the builder reserves nothing.", BuilderVersionsCall, since.UTC().Format(time.RFC3339), cause), + "indexer."+mcc.KindCallUnanswered) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("versions fill: the finding %s is not written: %v", BuilderVersionsCall, err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 5fbf0fe0..3f459836 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -2,6 +2,7 @@ package index import ( "context" + "errors" "fmt" "testing" "time" @@ -78,3 +79,81 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) { t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest) } } + +// Design 47 G §372: the first fill records, once, each module's highest published version over its artifacts as released +// and says versions-filled once; a builder_versions that does not answer holds it and, after five minutes, is the finding +// call-unanswered; a fill made is not made again. +func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + w := newFindings() + var filledSaid []meshcatalog.VersionsFilledEvent + ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, + SayVersionsFilled: func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error { + filledSaid = append(filledSaid, e) + return nil + }} + entries := map[string]meshcatalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{ + {Ref: "mesh-sdk:go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/mesh-sdk/go"}}, + {Ref: "mesh-sdk:ts", Package: &meshcatalog.ArtifactPackage{Ecosystem: "npm", Name: "@novox/mesh-sdk"}}}}, + "avahi": {Module: "avahi", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:image", Kind: "image"}}}, + "fresh": {Module: "fresh", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "fresh:image", Kind: "image"}}}, + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + up := false + ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) { + if !up { + return BuilderVersionsAnswer{}, errors.New("no responders") + } + pub := func(v, d string) BuilderVersion { + return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9"} + } + switch name { + case "git.novox.be/novox/mesh-sdk/go": + return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.1.14", "sha256:go14"), {Version: "0.1.15", State: "refused"}}}, nil + case "@novox/mesh-sdk": + return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.1.18", "sha256:ts18")}}, nil + case "avahi/image": + return BuilderVersionsAnswer{Name: name, Versions: []BuilderVersion{pub("0.2.3", "sha256:av")}}, nil + } + return BuilderVersionsAnswer{Name: name}, nil + } + if err := ix.FillVersions(ctx, ask); err == nil || len(filledSaid) != 0 { + t.Fatalf("a builder_versions unanswered filled: %v", err) + } + if len(w.Held()) != 0 { + t.Fatal("a call unanswered for no time wrote its finding") + } + now = now.Add(5 * time.Minute) + _ = ix.FillVersions(ctx, ask) + if h := w.Held(); len(h) != 1 || h[0].ID != BuilderVersionsCall { + t.Fatalf("the findings after five minutes: %+v", h) + } + up = true + if err := ix.FillVersions(ctx, ask); err != nil { + t.Fatal(err) + } + if len(w.Held()) != 0 { + t.Errorf("the finding stands after builder_versions answered: %+v", w.Held()) + } + if len(filledSaid) != 1 || filledSaid[0].Modules != 2 { + t.Fatalf("versions-filled said %+v", filledSaid) + } + sdk, _ := ModuleVersions(ctx, ix.Versions, "mesh-sdk") + if len(sdk) != 1 || sdk[0].Version != "0.1.18" || len(sdk[0].Artifacts) != 1 || sdk[0].Artifacts[0].Artifact != "mesh-sdk:ts" || sdk[0].Source != 9 { + t.Errorf("mesh-sdk's filled version: %+v (the higher of Go's 0.1.14 and npm's 0.1.18)", sdk) + } + if fresh, _ := ModuleVersions(ctx, ix.Versions, "fresh"); len(fresh) != 0 { + t.Errorf("a module with none published is filled: %+v", fresh) + } + if err := ix.FillVersions(ctx, ask); err != nil || len(filledSaid) != 1 { + t.Errorf("filled again: %v, said %d", err, len(filledSaid)) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 9b42127a..7e7c7280 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -173,7 +173,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 33863ef4..093f7de4 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -155,7 +155,8 @@ "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", - "seat:node-build-image.steps" + "seat:node-build-image.steps", + "mesh-delivery-builder.builder_versions" ], "own-secrets": { "broker": "${dir:state}/broker" From d9066aa1c6038d0c42267fdd2ebf416a19530232 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:07:45 +0100 Subject: [PATCH 05/13] indexer: the release's edge entries, outdated for kinds 1, 2 and 4, toolchain-agreed and version-delivered kept as heard, the verb input and its state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kind 3, kinds 5 and 6, the first fill's edge entries (§377) and the deletion of an input at its ask's step-ended are not built. Builds: design 47 G §374, §375, §354, A-68 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer-tools/main.go | 4 +- .../cmd/mesh-delivery-indexer/main.go | 8 + .../cmd/mesh-delivery-indexer/versions.go | 35 ++++ .../internal/index/candidates.go | 2 + .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/run_inputs.go | 181 ++++++++++++++++ .../internal/index/versions.go | 93 +++++++-- .../internal/index/versions_edges.go | 197 ++++++++++++++++++ .../internal/index/versions_test.go | 125 +++++++++++ .../mesh-delivery-indexer/manifest_test.go | 12 +- modules/mesh-delivery-indexer/module.json | 12 +- 12 files changed, 640 insertions(+), 33 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/run_inputs.go create mode 100644 modules/mesh-delivery-indexer/internal/index/versions_edges.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index b4bf550f..cb249524 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -288,7 +288,9 @@ func main() { tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)})) + tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, OwnBuildState: keptRead{stdio.State(index.OwnBuildStateName)}}), + index.InputTool(keptRead{stdio.State(index.InputsState)}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)})) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index f5827821..7d261d67 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -445,6 +446,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro SayFill: sayFill(conn, cred), // The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released. Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)}, + Inputs: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)}, SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. @@ -679,6 +681,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshbuilds.SubjectPackagePublished, func(m jetstream.Msg) { heardPackagePublished(ctx, ix, m, logger.Printf) }); err != nil { return err } + if err := hub.Handle(meshbuilds.SubjectToolchainAgreed, func(m jetstream.Msg) { heardToolchainAgreed(ctx, ix, m, logger.Printf) }); err != nil { + return err + } + if err := hub.Handle(meshwants.SubjectVersionDelivered, func(m jetstream.Msg) { heardVersionDelivered(ctx, ix, m, logger.Printf) }); err != nil { + return err + } if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index f8507c48..ef0bca4d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" @@ -59,6 +60,40 @@ func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.M _ = m.Ack() } +// heardToolchainAgreed keeps the newest toolchain-agreed of its language (A-68); one that does not decode is said and +// acknowledged, one that cannot be kept is left, so the bus gives it again. +func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var t meshbuilds.ToolchainAgreed + if err := json.Unmarshal(m.Data(), &t); err != nil { + logf("%s holds what is no toolchain-agreed, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.HearToolchainAgreed(ctx, t); err != nil { + logf("versions: the toolchain-agreed of %s could not be kept, so the bus gives it again: %v", t.Language, err) + _ = m.Nak() + return + } + _ = m.Ack() +} + +// heardVersionDelivered writes the delivered release's entries of kinds 2, 4 and 6 and keeps the newest delivered version of +// its module (design 47 G §374, A-68), as heardToolchainAgreed keeps its event. +func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var d meshwants.VersionDeliveredEvent + if err := json.Unmarshal(m.Data(), &d); err != nil { + logf("%s holds what is no version-delivered, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.HearVersionDelivered(ctx, d); err != nil { + logf("versions: the version-delivered of %s %s could not be kept, so the bus gives it again: %v", d.Module, d.Version, err) + _ = m.Nak() + return + } + _ = m.Ack() +} + // sayVersionsFilled says versions-filled (design 47 G §372) once, on the seat's stream that keeps it with no age limit. func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionsFilledEvent, string) error { return func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index a683c83b..9a19d11b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -113,6 +113,8 @@ type Indexer struct { SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error // SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing. SayVersionsFilled func(ctx context.Context, e meshcatalog.VersionsFilledEvent, id string) error + // Inputs is the state inputs (run_inputs.go, design 47 G §354): the inputs of the indexer's run asks; nil keeps none. + Inputs Store // Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads // what the first registrations are composed from (§8). Nil: no registration act is answered. Registry *Registry diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index a18efefe..49ef4603 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -910,7 +910,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, OwnBuildState: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, + Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, // A dry run writes no registration and asks no one. Registry: nil, SeedModules: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index e51ba1dc..73da47fe 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/run_inputs.go b/modules/mesh-delivery-indexer/internal/index/run_inputs.go new file mode 100644 index 00000000..647737ae --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/run_inputs.go @@ -0,0 +1,181 @@ +package index + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "slices" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The indexer's inputs (design 47 G §354): the input of the ask `run` the indexer's validation puts, kept in the old state +// section inputs, one key per input named by its write time and its sha256, read by the seat mesh-catalog's verb input (whose +// callers are the holders of the build seats) and by the tool indexer_inputs. A key older than a day is deleted at each new +// ask, judged over at most the 20 oldest keys. +const ( + InputsState = "inputs" + InputsBucket = Module + "_" + InputsState + InputsTool = "indexer_inputs" + // InputsKeptFor is how long an input is kept at the longest. + InputsKeptFor = 24 * time.Hour + // InputsSwept is how many of the oldest keys each new ask judges. + InputsSwept = 20 + inputsTime = "20060102T150405Z" +) + +// InputDigest is the sha256 of an input's exact bytes, as "sha256:" and its hex digits. +func InputDigest(body []byte) string { + sum := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// KeepInput keeps the exact bytes of a run ask's input under their write time and sha256, after sweeping the keys more than a +// day old among the 20 oldest; it answers the digest. The same bytes written again in the same second are one key. +func (ix *Indexer) KeepInput(ctx context.Context, body []byte) (string, error) { + if ix.Inputs == nil { + return "", errors.New("the state inputs is not kept here") + } + if err := ix.sweepInputs(ctx); err != nil { + ix.logf("inputs: the keys older than a day are not all deleted: %v", err) + } + digest := InputDigest(body) + key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest) + data, err := json.Marshal(base64.StdEncoding.EncodeToString(body)) + if err != nil { + return "", err + } + if err := ix.Inputs.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { + return "", err + } + return digest, nil +} + +// sweepInputs deletes, among the 20 oldest keys, those written more than a day ago. +func (ix *Indexer) sweepInputs(ctx context.Context) error { + all, err := ix.Inputs.Keys(ctx) + if err != nil { + return err + } + slices.Sort(all) // the write time leads the key, so the oldest come first + if len(all) > InputsSwept { + all = all[:InputsSwept] + } + var errs []error + for _, k := range all { + t, _, ok := meshcatalog.ParseInputsByTimeDigestKey(k) + at, perr := time.Parse(inputsTime, t) + if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor { + continue + } + if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found { + if err := ix.Inputs.Delete(ctx, k, rev); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +// InputKey is the key of the input of the digest, or none. +func InputKey(ctx context.Context, st Store, digest string) (string, error) { + all, err := st.Keys(ctx) + if err != nil { + return "", err + } + for _, k := range all { + if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest { + return k, nil + } + } + return "", nil +} + +// Input answers the verb input: the exact bytes (base64) of the input of the digest, its digest checked. +func Input(ctx context.Context, st Store, in meshcatalog.InputInput) (meshcatalog.InputAnswer, error) { + var out meshcatalog.InputAnswer + if in.Digest == "" { + return out, errors.New("name the input's sha256 (digest)") + } + key, err := InputKey(ctx, st, in.Digest) + if err != nil { + return out, fmt.Errorf("the state %s cannot be read: %w", InputsState, err) + } + raw, _, found, err := st.Get(ctx, key) + if key == "" || err != nil || !found { + return out, fmt.Errorf("no input of digest %s is kept", in.Digest) + } + text, err := meshcatalog.DecodeInputsByTimeDigestValue(raw) + if err != nil { + return out, fmt.Errorf("the input %s cannot be read: %w", key, err) + } + body, err := base64.StdEncoding.DecodeString(text) + if err != nil || InputDigest(body) != in.Digest { + return out, fmt.Errorf("the input kept under %s does not match its digest", key) + } + return meshcatalog.InputAnswer{Digest: in.Digest, Input: text}, nil +} + +// InputTool is the verb input as the tool mesh-catalog.input of the indexer's own runtime. +func InputTool(st Store) stdio.Tool { + return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbInput, Run: func(args map[string]any) (any, error) { + var in meshcatalog.InputInput + if raw, err := json.Marshal(args); err == nil { + _ = json.Unmarshal(raw, &in) + } + return Input(context.Background(), st, in) + }} +} + +// InputsAnswer is what indexer_inputs answers: each key kept with its time, digest and size, or one input's bytes. +type InputsAnswer struct { + Inputs []InputKept `json:"inputs"` +} + +// InputKept is one input kept. +type InputKept struct { + Key string `json:"key"` + Digest string `json:"digest"` + Bytes int `json:"bytes"` + Input string `json:"input,omitempty"` +} + +// InputsReadTool is indexer_inputs (class read): every input the indexer keeps, with its key, digest and size; with a digest, +// that input whole (base64). +func InputsReadTool(st Store) stdio.Tool { + return stdio.Tool{Name: InputsTool, Description: "Read the indexer's inputs of the ask run (design 47 G section 354): each key (write time and sha256) with its size; with a digest, that input's exact bytes, base64.", + Input: map[string]any{"type": "object", "properties": map[string]any{"digest": map[string]any{"type": "string", "description": "one input's sha256; none lists every key"}}}, + Run: func(args map[string]any) (any, error) { + ctx := context.Background() + want, _ := args["digest"].(string) + all, err := st.Keys(ctx) + if err != nil { + return nil, err + } + slices.Sort(all) + out := InputsAnswer{Inputs: []InputKept{}} + for _, k := range all { + _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k) + if !ok || (want != "" && d != want) { + continue + } + raw, _, found, err := st.Get(ctx, k) + if err != nil || !found { + continue + } + text, _ := meshcatalog.DecodeInputsByTimeDigestValue(raw) + kept := InputKept{Key: k, Digest: d, Bytes: base64.StdEncoding.DecodedLen(len(text))} + if want != "" { + kept.Input = text + } + out.Inputs = append(out.Inputs, kept) + } + return out, nil + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 8163a690..154013e4 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -1,6 +1,7 @@ package index import ( + "cmp" "context" "encoding/json" "errors" @@ -32,8 +33,9 @@ const ( // published is what a not yet released version holds: each artifact published at it, by reference. type published struct { - Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"` - Source int64 `json:"source,omitempty"` + Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"` + Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"` + Source int64 `json:"source,omitempty"` } // versionParts is MAJOR.MINOR.PATCH, read; false for any other form. @@ -113,10 +115,14 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil || found { return err // released already: heard again } - want, err := ix.artifactsOf(ctx, p.Module) + entry, err := ix.entryOf(ctx, p.Module) if err != nil { return err } + var want []string + for _, a := range entry.Artifacts { + want = append(want, a.Ref) + } key := PendingPrefix + meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version) var have published err = write(ctx, ix.OwnBuildState, key, func(raw []byte, found bool) ([]byte, bool, error) { @@ -131,6 +137,11 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag } have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest} have.Source = p.Source + for _, d := range p.Dependencies { + if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) { + have.Dependencies = append(have.Dependencies, d) + } + } data, err := json.Marshal(have) return data, false, err }) @@ -142,41 +153,36 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag return nil } } - return ix.releaseVersion(ctx, p, key, want, have) + return ix.releaseVersion(ctx, p, key, entry, have) } -// artifactsOf are the artifact references the module's entry in the newest trunk revision names. -func (ix *Indexer) artifactsOf(ctx context.Context, module string) ([]string, error) { +// entryOf is the module's entry in the newest trunk revision. +func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) { if ix.Revisions == nil { - return nil, errors.New("the catalog graph is not kept here: no entry names the module's artifacts") + return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts") } rev, found, err := newestTrunk(ctx, ix.Revisions) if err != nil { - return nil, err + return meshcatalog.CatalogEntry{}, err } if !found { - return nil, errors.New("the indexer keeps no trunk revision yet") + return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") } _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) if err != nil { - return nil, err + return meshcatalog.CatalogEntry{}, err } for _, e := range entries { - if e.Module != module { - continue + if e.Module == module { + return e, nil } - var refs []string - for _, a := range e.Artifacts { - refs = append(refs, a.Ref) - } - return refs, nil } - return nil, fmt.Errorf("the trunk holds no module %s", module) + return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module) } // releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions, // and forgets the artifacts kept as published. -func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, want []string, have published) error { +func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error { key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version) kept, err := ModuleVersions(ctx, ix.Versions, p.Module) if err != nil { @@ -185,9 +191,11 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0 at := ix.now().UTC() v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late} - for _, ref := range slices.Sorted(slices.Values(want)) { - v.Artifacts = append(v.Artifacts, have.Artifacts[ref]) + for _, a := range entry.Artifacts { + v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref]) } + slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) }) + v.Edges = releaseEdges(entry, have.Dependencies) data, err := json.Marshal(v) if err != nil { return err @@ -270,3 +278,46 @@ func VersionsTool(st Store) stdio.Tool { return Versions(context.Background(), st, in) }} } + +// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374). +const notYetDelivered = "not yet delivered" + +// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build +// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's +// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here. +func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge { + word := func(kind int64, part string) string { + for _, e := range entry.Edges { + if e.Kind == kind && (e.Part == part || e.From == part) { + return e.Word + } + } + return "" + } + var out []meshcatalog.VersionEdge + for _, d := range deps { + out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)}) + } + for _, e := range entry.Edges { + if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 { + continue + } + part := cmp.Or(e.Part, e.From) + if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) { + out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered}) + } + } + return out +} + +// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions +// state and the heard toolchain and delivered keys of the own state. +func OutdatedTool(ix *Indexer) stdio.Tool { + return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) { + var in meshcatalog.OutdatedInput + if raw, err := json.Marshal(args); err == nil { + _ = json.Unmarshal(raw, &in) + } + return ix.Outdated(context.Background(), in) + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go new file mode 100644 index 00000000..5de415e7 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -0,0 +1,197 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" +) + +// What outdated compares against is kept as heard, in the indexer's own state (design 47 G A-68): toolchain. the newest +// toolchain-agreed per language, delivered. the newest version-delivered per module. +const ( + ToolchainPrefix = "toolchain." + DeliveredPrefix = "delivered." +) + +// keepNewest writes body at key of the own-build state unless it holds a value whose version is higher; version of the body +// is read by newer, which says whether the heard one is not older than the kept. +func (ix *Indexer) keepNewest(ctx context.Context, key string, body any, newer func(kept []byte) bool) error { + if ix.OwnBuildState == nil { + return errors.New("the own state is not kept here") + } + data, err := json.Marshal(body) + if err != nil { + return err + } + return write(ctx, ix.OwnBuildState, key, func(kept []byte, found bool) ([]byte, bool, error) { + if found && !newer(kept) { + return nil, false, nil + } + return data, false, nil + }) +} + +// HearToolchainAgreed keeps the newest toolchain-agreed of a language (A-68, §375): the one heard last. +func (ix *Indexer) HearToolchainAgreed(ctx context.Context, t meshbuilds.ToolchainAgreed) error { + if t.Language == "" { + return errors.New("a toolchain-agreed names no language") + } + return ix.keepNewest(ctx, ToolchainPrefix+t.Language, t, func([]byte) bool { return true }) +} + +// HearVersionDelivered takes one version-delivered (design 47 G §374, §376, A-68): the entries of kinds 2, 4 and 6 of the +// delivered release, which read "not yet delivered", are written once with each dependency's newest delivered version and +// part digest; then the delivered version is kept as the module's newest. +func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.VersionDeliveredEvent) error { + if ix.Versions == nil || ix.OwnBuildState == nil { + return errors.New("the state versions is not kept here") + } + if _, ok := versionParts(d.Version); !ok { + return fmt.Errorf("the version-delivered of %s names the version %q, which is no MAJOR.MINOR.PATCH", d.Module, d.Version) + } + key := meshcatalog.VersionsByModuleVersionKey(d.Module, d.Version) + err := write(ctx, ix.Versions, key, func(raw []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil // a version the catalog does not hold (a build of another branch) has no entries to write + } + v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw) + if err != nil { + return nil, false, err + } + changed := false + for i, e := range v.Edges { + if e.Version != notYetDelivered { + continue + } + dep, ok := ix.deliveredOf(ctx, strings.SplitN(e.Part, ":", 2)[0]) + if !ok { + continue // the dependency is not delivered yet: the entry reads so until this release's own delivery finds it + } + v.Edges[i].Version, v.Edges[i].Digest = dep.Version, dep.Parts[e.Part] + changed = true + } + if !changed { + return nil, false, nil + } + data, err := json.Marshal(v) + return data, false, err + }) + if err != nil { + return err + } + return ix.keepNewest(ctx, DeliveredPrefix+d.Module, d, func(kept []byte) bool { + var k meshwants.VersionDeliveredEvent + return json.Unmarshal(kept, &k) != nil || compareVersions(d.Version, k.Version) >= 0 + }) +} + +// deliveredOf is the newest version-delivered kept of a module. +func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.VersionDeliveredEvent, bool) { + var d meshwants.VersionDeliveredEvent + raw, _, found, err := ix.OwnBuildState.Get(ctx, DeliveredPrefix+module) + if err != nil || !found || json.Unmarshal(raw, &d) != nil { + return d, false + } + return d, true +} + +// Outdated answers the verb outdated (design 47 G §375) for the edge kinds 1, 2 and 4: every entry of each module's newest +// released version that is behind by the part's digest: kind 1, the recorded digest differs from the part's digest in the +// dependency's newest released version; kinds 2 and 4, from the dependency's newest version-delivered. A stale edge (kind 1) +// has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that left the digest as it +// was makes nothing behind. +func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (meshcatalog.OutdatedAnswer, error) { + out := meshcatalog.OutdatedAnswer{Behind: []meshcatalog.Outdated{}} + if ix.Versions == nil || ix.OwnBuildState == nil { + return out, errors.New("the state versions is not kept here") + } + all, err := ix.Versions.Keys(ctx) + if err != nil { + return out, err + } + modules := map[string]bool{} + for _, k := range all { + if m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k); ok { + modules[m] = true + } + } + newest := map[string]meshcatalog.VersionEntry{} + for m := range modules { + vs, err := ModuleVersions(ctx, ix.Versions, m) + if err != nil { + return out, err + } + for _, v := range vs { + if !v.Late { + newest[m] = v + break + } + } + } + for _, m := range sortedKeys(newest) { + v := newest[m] + for _, e := range v.Edges { + dep := strings.SplitN(e.Part, ":", 2)[0] + if in.Dependency != "" && in.Dependency != e.Part { + continue + } + var latest, digest string + switch e.Kind { + case 1: + n, ok := newest[dep] + if !ok { + continue + } + for _, a := range n.Artifacts { + if a.Artifact == e.Part { + latest, digest = n.Version, a.Digest + } + } + case 2, 4: + d, ok := ix.deliveredOf(ctx, dep) + if !ok || e.Version == notYetDelivered { + continue + } + latest, digest = d.Version, d.Parts[e.Part] + default: + continue + } + if digest == "" || digest == e.Digest { + continue + } + o := meshcatalog.Outdated{Module: m, Version: v.Version, Kind: e.Kind, Recorded: e.Version + " " + e.Digest, Newest: latest + " " + digest, + Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: ix.firstBranch(ctx, dep)}} + if e.Stale { + o.Act = "rebuild" + } + out.Behind = append(out.Behind, o) + } + } + return out, nil +} + +// firstBranch is the first watched branch of a module's registration, which its released versions are of. +func (ix *Indexer) firstBranch(ctx context.Context, module string) string { + if ix.Registry != nil { + if r, found, err := ix.Registry.Registration(ctx, module); err == nil && found && len(r.Branches) > 0 { + return r.Branches[0] + } + } + return "" +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 3f459836..7b5f1ebf 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" + "strings" "testing" "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" ) // Design 47 G §349: a version is released once every artifact the module's entry names is published at it, said once as @@ -157,3 +159,126 @@ func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) { t.Errorf("filled again: %v, said %d", err, len(filledSaid)) } } + +// Design 47 G §374, §375, A-68: a release records, per edge, the part it was built against (kind 1, from package-published's +// dependencies) or, once delivered, the delivered version it was ordered after (kinds 2 and 4); outdated answers an edge behind +// by the part's digest alone, with the act rebuild only for a stale edge; a newer version that left the digest as it was +// makes nothing behind. +func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Revisions: newMemObjects(1 << 20)} + entries := map[string]meshcatalog.CatalogEntry{ + "lib": {Module: "lib", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}}, + "dep": {Module: "dep", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "dep:program"}}}, + "app": {Module: "app", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}}, + Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}, + {Kind: 4, From: "dep:program", To: "app:program", Part: "dep:program", Word: "provision"}}}, + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + pub := func(module, version, digest string, deps ...meshbuilds.Dependency) { + t.Helper() + if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: module, Artifact: module + ":" + map[string]string{"lib": "pkg", "dep": "program", "app": "program"}[module], Digest: digest, Version: version, Dependencies: deps}); err != nil { + t.Fatal(err) + } + } + behind := func() []meshcatalog.Outdated { + t.Helper() + a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{}) + if err != nil { + t.Fatal(err) + } + return a.Behind + } + pub("lib", "0.1.1", "d1") + pub("dep", "0.1.1", "x1") + pub("app", "0.1.1", "a1", meshbuilds.Dependency{Kind: 1, Part: "lib:pkg", Version: "0.1.1", Digest: "d1"}) + v, _ := ModuleVersions(ctx, ix.Versions, "app") + if len(v[0].Edges) != 2 || v[0].Edges[0].Kind != 1 || !v[0].Edges[0].Stale || v[0].Edges[0].Word != "imports" || v[0].Edges[1].Version != "not yet delivered" { + t.Fatalf("the release's edges: %+v", v[0].Edges) + } + pub("lib", "0.1.2", "d1") // a newer version, the part's digest as it was + if b := behind(); len(b) != 0 { + t.Fatalf("a newer version that left the digest as it was is behind: %+v", b) + } + pub("lib", "0.1.3", "d3") + if b := behind(); len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Act != "rebuild" || b[0].Dependency.Artifact != "lib:pkg" || !strings.Contains(b[0].Newest, "d3") || !strings.Contains(b[0].Recorded, "d1") { + t.Fatalf("behind after lib's new digest: %+v", b) + } + // delivery: dep 0.1.1 is delivered, then app 0.1.1 writes its kind 4 entry; dep 0.1.2 delivered later is behind, an ordering edge + deliver := func(module, version string, parts map[string]string) { + t.Helper() + if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: module, Version: version, Parts: parts}); err != nil { + t.Fatal(err) + } + } + deliver("dep", "0.1.1", map[string]string{"dep:program": "x1"}) + deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) + v, _ = ModuleVersions(ctx, ix.Versions, "app") + if e := v[0].Edges[1]; e.Version != "0.1.1" || e.Digest != "x1" { + t.Fatalf("the kind 4 entry after the first delivery: %+v", e) + } + deliver("dep", "0.1.2", map[string]string{"dep:program": "x2"}) + deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) // delivered again: written once, not changed + v, _ = ModuleVersions(ctx, ix.Versions, "app") + if e := v[0].Edges[1]; e.Version != "0.1.1" { + t.Errorf("an entry written at the first delivery changed: %+v", e) + } + b := behind() + if len(b) != 2 || b[1].Kind != 4 || b[1].Act != "" || b[1].Dependency.Artifact != "dep:program" { + t.Fatalf("behind after dep's second delivery: %+v", b) + } + // the newest toolchain-agreed per language is kept as heard + if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil { + t.Fatal(err) + } + if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil { + t.Fatal(err) + } + if raw, _, found, _ := ix.OwnBuildState.Get(ctx, ToolchainPrefix+"go"); !found || !strings.Contains(string(raw), "1.26") { + t.Errorf("toolchain.go holds %s", raw) + } +} + +// Design 47 G §354: an input is kept under its write time and sha256, read back by the verb input with its digest checked; +// the keys older than a day among the 20 oldest are deleted at each new keep. +func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }} + d1, err := ix.KeepInput(ctx, []byte("first")) + if err != nil { + t.Fatal(err) + } + a, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}) + if err != nil || a.Digest != d1 || a.Input != "Zmlyc3Q=" { + t.Fatalf("the input read back: %+v, %v", a, err) + } + if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: "sha256:none"}); err == nil { + t.Error("an input never kept is answered") + } + now = now.Add(23 * time.Hour) + if _, err := ix.KeepInput(ctx, []byte("second")); err != nil { + t.Fatal(err) + } + if keys, _ := ix.Inputs.Keys(ctx); len(keys) != 2 { + t.Fatalf("inside a day: %v", keys) + } + now = now.Add(2 * time.Hour) + d3, _ := ix.KeepInput(ctx, []byte("third")) + keys, _ := ix.Inputs.Keys(ctx) + if len(keys) != 2 { + t.Fatalf("after a day the first stays: %v", keys) + } + if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil { + t.Error("the input a day old is still answered") + } + if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d3}); err != nil { + t.Error(err) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 7e7c7280..2ac33f79 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -125,7 +125,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, - {Name: index.VersionsState, History: 1, MaxMiB: 64}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 + {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.InputsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, @@ -159,9 +159,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-wants.version-delivered", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -196,8 +196,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool) // design 47 G §667, §8 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool) // design 47 G §667, §8 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. @@ -301,7 +301,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools // bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own. if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions"}) { + !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions", "outdated", "input"}) { t.Errorf("claims = %+v, want mesh-catalog with its seven verbs (entry: design 47 G §675)", m.Claims) } for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 093f7de4..f9427b76 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -20,7 +20,9 @@ "policy", "watch", "unwatch", - "versions" + "versions", + "outdated", + "input" ] } ], @@ -69,7 +71,8 @@ "indexer_entries", "indexer_proposals", "indexer_proposal_drop", - "indexer_seed_again" + "indexer_seed_again", + "indexer_inputs" ], "consumes": [ "go-package-registry.published", @@ -81,6 +84,8 @@ "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-builds.package-published", + "mesh-builds.toolchain-agreed", + "mesh-wants.version-delivered", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.source-moved", @@ -110,7 +115,8 @@ { "name": "entry-index", "history": 1, "max-mib": 8 }, { "name": "registrations", "history": 1 }, { "name": "proposals", "history": 1, "max-mib": 4 }, - { "name": "versions", "history": 1, "max-mib": 64 } + { "name": "versions", "history": 1, "max-mib": 64 }, + { "name": "inputs", "history": 1 } ], "streams": [ { "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 }, From bec5e063bc158dc0fc3a20591b9efd3b3f15b731 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:10:36 +0100 Subject: [PATCH 06/13] indexer: what it heard (steps-since, call-since, toolchain, delivered, pending) lives in the state heard, each key part escaped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The own-build state keeps its one key. Builds: design 47 G A-80, §73, §375 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 7 ++-- .../internal/index/candidates.go | 4 +++ .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/seed_steps.go | 21 +++++------ .../internal/index/seed_steps_test.go | 4 +-- .../internal/index/versions.go | 15 ++++---- .../internal/index/versions_edges.go | 19 +++++----- .../internal/index/versions_fill.go | 13 +++---- .../internal/index/versions_test.go | 35 ++++++++++++++++--- 11 files changed, 80 insertions(+), 44 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index cb249524..3311d150 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -289,7 +289,7 @@ func main() { // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), - index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, OwnBuildState: keptRead{stdio.State(index.OwnBuildStateName)}}), + index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}}), index.InputTool(keptRead{stdio.State(index.InputsState)}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)})) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 7d261d67..00285e74 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -399,6 +399,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // appears; until then indexer_requirements answers none, and the builder's floor waits on its answer. requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf) // The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes. + // The state heard (A-80): the forge messages handled (key forge) and what the indexer heard and compares later (the time a call is + // unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version). + heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "what the indexer heard is not kept", logger.Printf)} ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf) // The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it // appears; until then a take-in's new manifests are kept for their judging in its memory alone. @@ -438,7 +441,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, + OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). @@ -746,7 +749,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) }) // What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds // (issue 1790): until then no message is written there, and the rest of the indexer runs. - heard := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)} + heard := heardStore // Each change of the states its readers read is said on its views family (ADR 0615 §5), from the mark each state was // said up to; a first view (the tools bundle) answers at that mark. startViews(ctx, js, cred, marks, writer, diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 9a19d11b..aba8ab7c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -190,6 +190,10 @@ type Indexer struct { ownWhy string // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. OwnBuildState Store + // Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since), + // toolchain., delivered., pending.., acted.., each key part escaped as a name + // (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled. + Heard Store judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) // Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a // pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none. diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 49ef4603..f2c65992 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -908,7 +908,7 @@ func (ix *Indexer) dry() *dryRun { // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, - OwnBuildState: nil, underSeats: true, + OwnBuildState: nil, Heard: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, // A dry run writes no registration and asks no one. diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 73da47fe..43a0003c 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 1453433f..a4bd5faf 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -11,6 +11,7 @@ import ( mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/seatdef" ) @@ -170,25 +171,25 @@ func namedPaths(e meshcatalog.CatalogEntry, name string) []string { return e.Inputs.Paths[ArtifactRefOf(e.Module, name)] } -// StepsSincePrefix opens the keys of the own-build state (OwnBuildState) that keep, by build seat, since when its steps +// StepsSincePrefix opens the keys of the state heard (Heard, A-80) that keep, by build seat, since when its steps // has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past // five minutes. const StepsSincePrefix = "steps-since." // stepsUnansweredSince is the time the seat first failed to answer. func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time { - return ix.unansweredSince(ctx, StepsSincePrefix+seat, now) + return ix.unansweredSince(ctx, StepsSincePrefix+keys.EscapeName(seat), now) } // stepsAnswered forgets the time a seat that answers was unanswered since. func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) { - ix.answered(ctx, StepsSincePrefix+seat) + ix.answered(ctx, StepsSincePrefix+keys.EscapeName(seat)) } -// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's own state (the first +// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's state heard (the first // failure writes now); where no state is kept it is held in memory. func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time { - if ix.OwnBuildState == nil { + if ix.Heard == nil { ix.mu.Lock() defer ix.mu.Unlock() if ix.stepsSince == nil { @@ -199,12 +200,12 @@ func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Tim } return ix.stepsSince[key] } - if v, _, found, err := ix.OwnBuildState.Get(ctx, key); err == nil && found { + if v, _, found, err := ix.Heard.Get(ctx, key); err == nil && found { if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil { return t } } - if err := ix.OwnBuildState.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil { + if err := ix.Heard.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil { ix.logf("the time since %s is unanswered is not kept: %v", key, err) } return now @@ -212,14 +213,14 @@ func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Tim // answered forgets the time the call kept at key was unanswered since. func (ix *Indexer) answered(ctx context.Context, key string) { - if ix.OwnBuildState == nil { + if ix.Heard == nil { ix.mu.Lock() delete(ix.stepsSince, key) ix.mu.Unlock() return } - if _, rev, found, err := ix.OwnBuildState.Get(ctx, key); err == nil && found { - if err := ix.OwnBuildState.Delete(ctx, key, rev); err != nil { + if _, rev, found, err := ix.Heard.Get(ctx, key); err == nil && found { + if err := ix.Heard.Delete(ctx, key, rev); err != nil { ix.logf("the time since %s is unanswered is not forgotten: %v", key, err) } } diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index f6c14b0e..a903f7ff 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -20,7 +20,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { ctx := context.Background() now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) w := newFindings() - ix := &Indexer{Findings: w, OwnBuildState: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} + ix := &Indexer{Findings: w, Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }} fp := "src2:" + strings.Repeat("a", 64) entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}}, @@ -78,7 +78,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Fatal("a call unanswered for no time wrote its finding") } now = now.Add(5 * time.Minute) - ix = &Indexer{Findings: w, OwnBuildState: ix.OwnBuildState, Revisions: ix.Revisions, Now: func() time.Time { return now }} // a restart keeps the time + ix = &Indexer{Findings: w, Heard: ix.Heard, Revisions: ix.Revisions, Now: func() time.Time { return now }} // a restart keeps the time if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 154013e4..8327575a 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -26,8 +26,9 @@ const ( // VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them // go, never its highest of any MAJOR.MINOR. VersionsKept = 100 - // PendingPrefix opens the keys of the own-build state (OwnBuildState) that keep, per module and version not yet - // released, the artifacts published at it so far (A-68's state, the one of A-66). + // PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the + // artifacts published at it so far, each key part escaped as a name (A-80, §73). + PendingPrefix = "pending." ) @@ -101,7 +102,7 @@ func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog // and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A // build of a watched branch other than the registration's first releases none (§371). func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error { - if ix.Versions == nil || ix.OwnBuildState == nil { + if ix.Versions == nil || ix.Heard == nil { return errors.New("the state versions is not kept here") } if _, ok := versionParts(p.Version); !ok { @@ -123,9 +124,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag for _, a := range entry.Artifacts { want = append(want, a.Ref) } - key := PendingPrefix + meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version) + key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version) var have published - err = write(ctx, ix.OwnBuildState, key, func(raw []byte, found bool) ([]byte, bool, error) { + err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) { have = published{} if found { if err := json.Unmarshal(raw, &have); err != nil { @@ -249,10 +250,10 @@ func (ix *Indexer) trimVersions(ctx context.Context, module string) error { // forgetPublished removes the artifacts kept as published at a version released. func (ix *Indexer) forgetPublished(ctx context.Context, key string) error { - if _, rev, found, err := ix.OwnBuildState.Get(ctx, key); err != nil || !found { + if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found { return err } else { - return ix.OwnBuildState.Delete(ctx, key, rev) + return ix.Heard.Delete(ctx, key, rev) } } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index 5de415e7..3574577a 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -11,26 +11,27 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/keys" ) -// What outdated compares against is kept as heard, in the indexer's own state (design 47 G A-68): toolchain. the newest +// What outdated compares against is kept as heard, in the indexer's state heard (design 47 G A-68): toolchain. the newest // toolchain-agreed per language, delivered. the newest version-delivered per module. const ( ToolchainPrefix = "toolchain." DeliveredPrefix = "delivered." ) -// keepNewest writes body at key of the own-build state unless it holds a value whose version is higher; version of the body +// keepNewest writes body at key of the state heard unless it holds a value whose version is higher; version of the body // is read by newer, which says whether the heard one is not older than the kept. func (ix *Indexer) keepNewest(ctx context.Context, key string, body any, newer func(kept []byte) bool) error { - if ix.OwnBuildState == nil { + if ix.Heard == nil { return errors.New("the own state is not kept here") } data, err := json.Marshal(body) if err != nil { return err } - return write(ctx, ix.OwnBuildState, key, func(kept []byte, found bool) ([]byte, bool, error) { + return write(ctx, ix.Heard, key, func(kept []byte, found bool) ([]byte, bool, error) { if found && !newer(kept) { return nil, false, nil } @@ -43,14 +44,14 @@ func (ix *Indexer) HearToolchainAgreed(ctx context.Context, t meshbuilds.Toolcha if t.Language == "" { return errors.New("a toolchain-agreed names no language") } - return ix.keepNewest(ctx, ToolchainPrefix+t.Language, t, func([]byte) bool { return true }) + return ix.keepNewest(ctx, ToolchainPrefix+keys.EscapeName(t.Language), t, func([]byte) bool { return true }) } // HearVersionDelivered takes one version-delivered (design 47 G §374, §376, A-68): the entries of kinds 2, 4 and 6 of the // delivered release, which read "not yet delivered", are written once with each dependency's newest delivered version and // part digest; then the delivered version is kept as the module's newest. func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.VersionDeliveredEvent) error { - if ix.Versions == nil || ix.OwnBuildState == nil { + if ix.Versions == nil || ix.Heard == nil { return errors.New("the state versions is not kept here") } if _, ok := versionParts(d.Version); !ok { @@ -86,7 +87,7 @@ func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.Version if err != nil { return err } - return ix.keepNewest(ctx, DeliveredPrefix+d.Module, d, func(kept []byte) bool { + return ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(d.Module), d, func(kept []byte) bool { var k meshwants.VersionDeliveredEvent return json.Unmarshal(kept, &k) != nil || compareVersions(d.Version, k.Version) >= 0 }) @@ -95,7 +96,7 @@ func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.Version // deliveredOf is the newest version-delivered kept of a module. func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.VersionDeliveredEvent, bool) { var d meshwants.VersionDeliveredEvent - raw, _, found, err := ix.OwnBuildState.Get(ctx, DeliveredPrefix+module) + raw, _, found, err := ix.Heard.Get(ctx, DeliveredPrefix+keys.EscapeName(module)) if err != nil || !found || json.Unmarshal(raw, &d) != nil { return d, false } @@ -109,7 +110,7 @@ func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.Ve // was makes nothing behind. func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (meshcatalog.OutdatedAnswer, error) { out := meshcatalog.OutdatedAnswer{Behind: []meshcatalog.Outdated{}} - if ix.Versions == nil || ix.OwnBuildState == nil { + if ix.Versions == nil || ix.Heard == nil { return out, errors.New("the state versions is not kept here") } all, err := ix.Versions.Keys(ctx) diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go index b52537f4..4e4e8367 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -20,12 +20,13 @@ import ( const ( // BuilderVersionsCall is the tool the manifest names in its invokes (until that tool goes). BuilderVersionsCall = "mesh-delivery-builder.builder_versions" - // VersionsFilledKey is the key of the own-build state that says the fill is done, so it is not made again. + // VersionsFilledKey is the key of the state heard that says the fill is done, so it is not made again. VersionsFilledKey = "versions-filled" - // versionsFillSince is the own-build key of the time builder_versions is first unanswered. - versionsFillSince = "call-since." + BuilderVersionsCall ) +// versionsFillSince is the key of the state heard of the time builder_versions is first unanswered. +var versionsFillSince = "call-since." + keys.EscapeName(BuilderVersionsCall) + // BuilderVersion is one version of a name in the answer of builder_versions (its fields the fill reads). type BuilderVersion struct { Version string `json:"version"` @@ -57,14 +58,14 @@ func versionNameOf(module string, a meshcatalog.Artifact) string { // is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding // call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers. func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { - if ix.Versions == nil || ix.OwnBuildState == nil { + if ix.Versions == nil || ix.Heard == nil { return errors.New("the state versions is not kept here") } if !ix.versionsFilling.CompareAndSwap(false, true) { return nil // one fill runs at a time } defer ix.versionsFilling.Store(false) - if _, _, done, err := ix.OwnBuildState.Get(ctx, VersionsFilledKey); err != nil || done { + if _, _, done, err := ix.Heard.Get(ctx, VersionsFilledKey); err != nil || done { return err } if ix.Revisions == nil { @@ -149,7 +150,7 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { return fmt.Errorf("versions-filled is not said: %w", err) } } - if err := ix.OwnBuildState.Create(ctx, VersionsFilledKey, []byte(now.Format(time.RFC3339Nano))); err != nil && !errors.Is(err, ErrConflict) { + if err := ix.Heard.Create(ctx, VersionsFilledKey, []byte(now.Format(time.RFC3339Nano))); err != nil && !errors.Is(err, ErrConflict) { return err } ix.logf("versions: filled for %d modules from builder_versions, versions-filled said", filled) diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 7b5f1ebf..75caf1f8 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "testing" "time" @@ -20,7 +21,7 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) { ctx := context.Background() now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) var said []meshcatalog.VersionReleasedEvent - ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, + ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, SayVersionReleased: func(_ context.Context, e meshcatalog.VersionReleasedEvent, id string) error { said = append(said, e) return nil @@ -50,7 +51,7 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) { if len(v) != 1 || len(v[0].Artifacts) != 2 || v[0].Source != 7 || v[0].Late || len(said) != 1 || said[0].Version != "0.1.5" { t.Fatalf("released %+v, said %+v", v, said) } - if keys, _ := ix.OwnBuildState.Keys(ctx); len(keys) != 0 { + if keys, _ := ix.Heard.Keys(ctx); len(keys) != 0 { t.Errorf("the artifacts kept as published stay after the release: %v", keys) } // a late older version is recorded and not said @@ -90,7 +91,7 @@ func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) { now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) w := newFindings() var filledSaid []meshcatalog.VersionsFilledEvent - ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, + ix := &Indexer{Versions: newMem(), Heard: newMem(), Findings: w, Revisions: newMemObjects(1 << 20), Now: func() time.Time { return now }, SayVersionsFilled: func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error { filledSaid = append(filledSaid, e) return nil @@ -166,7 +167,7 @@ func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) { // makes nothing behind. func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) { ctx := context.Background() - ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Revisions: newMemObjects(1 << 20)} + ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)} entries := map[string]meshcatalog.CatalogEntry{ "lib": {Module: "lib", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}}, "dep": {Module: "dep", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "dep:program"}}}, @@ -240,7 +241,7 @@ func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) { if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil { t.Fatal(err) } - if raw, _, found, _ := ix.OwnBuildState.Get(ctx, ToolchainPrefix+"go"); !found || !strings.Contains(string(raw), "1.26") { + if raw, _, found, _ := ix.Heard.Get(ctx, ToolchainPrefix+"go"); !found || !strings.Contains(string(raw), "1.26") { t.Errorf("toolchain.go holds %s", raw) } } @@ -282,3 +283,27 @@ func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) { t.Error(err) } } + +// Design 47 G A-80, §73: what the indexer heard lives in the state heard, each key part escaped as a name, so a dotted module +// name parses and nothing of it is written to the own-build state. +func TestWhatIsHeardIsKeptEscapedInTheStateHeard(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Versions: newMem(), OwnBuildState: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)} + if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "a.b", Version: "1.2.3"}); err != nil { + t.Fatal(err) + } + if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil { + t.Fatal(err) + } + if _, ok := ix.deliveredOf(ctx, "a.b"); !ok { + t.Error("the delivered version of a dotted module is not read back") + } + got, _ := ix.Heard.Keys(ctx) + slices.Sort(got) + if want := []string{"delivered.a=2eb", "toolchain.go"}; !slices.Equal(got, want) { + t.Errorf("the keys of heard are %v, want %v", got, want) + } + if own, _ := ix.OwnBuildState.Keys(ctx); len(own) != 0 { + t.Errorf("the own-build state holds %v", own) + } +} From c6af94fe58d254cb9f564fe23ba50424fcd2bc59 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:13:38 +0100 Subject: [PATCH 07/13] indexer: outdated for kinds 3, 5 and 6; the acts of wait-ended kept as heard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-78 a, b, A-80, §374, §375 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 3 + .../cmd/mesh-delivery-indexer/versions.go | 16 ++ .../internal/index/candidates.go | 4 +- .../internal/index/versions_edges.go | 204 ++++++++++++++++-- .../internal/index/versions_test.go | 104 +++++++++ .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + 8 files changed, 319 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 3311d150..65227aab 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -289,7 +289,7 @@ func main() { // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), - index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}}), + index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputTool(keptRead{stdio.State(index.InputsState)}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)})) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 00285e74..8b9e0f39 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -690,6 +690,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshwants.SubjectVersionDelivered, func(m jetstream.Msg) { heardVersionDelivered(ctx, ix, m, logger.Printf) }); err != nil { return err } + if err := hub.Handle(meshwants.SubjectWaitEnded, func(m jetstream.Msg) { heardWaitEnded(ctx, ix, m, logger.Printf) }); err != nil { + return err + } if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index ef0bca4d..1f624d09 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -94,6 +94,22 @@ func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.M _ = m.Ack() } +// heardWaitEnded keeps the act of an acting part's ended wait for kind 5 (A-78 a), as heardToolchainAgreed keeps its event. +func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var w meshwants.WaitEndedEvent + if err := json.Unmarshal(m.Data(), &w); err != nil { + logf("%s holds what is no wait-ended, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.HearWaitEnded(ctx, w); err != nil { + logf("versions: the wait-ended of %s for %s could not be kept, so the bus gives it again: %v", w.Part, w.Module, err) + _ = m.Nak() + return + } + _ = m.Ack() +} + // sayVersionsFilled says versions-filled (design 47 G §372) once, on the seat's stream that keeps it with no age limit. func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionsFilledEvent, string) error { return func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index aba8ab7c..608f8c1e 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -193,8 +193,8 @@ type Indexer struct { // Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since), // toolchain., delivered., pending.., acted.., each key part escaped as a name // (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled. - Heard Store - judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) + Heard Store + judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) // Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a // pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none. Message func(ctx context.Context, repository, merge string, number int) ([]byte, error) diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index 3574577a..d99a61f8 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -1,12 +1,14 @@ package index import ( + "cmp" "context" "encoding/json" "errors" "fmt" "slices" "strings" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" @@ -19,8 +21,59 @@ import ( const ( ToolchainPrefix = "toolchain." DeliveredPrefix = "delivered." + // ActedPrefix opens acted.., the newest wait-ended of an acting part's wait for a dependent, ran or released + // (A-78 a), each key part escaped as a name (A-80, §73). + ActedPrefix = "acted." ) +// Acted is what an acted key holds: the build the dependent was at when the acting part acted for it (the wait's version, §374 kind 5). +type Acted struct { + V int64 `json:"v"` + Part string `json:"part"` + Step string `json:"step"` + Build string `json:"build"` + How string `json:"how"` + At time.Time `json:"at"` +} + +// HearWaitEnded keeps, for kind 5 (§374, §375, A-78 a), the newest wait-ended of an acting step that ended ran or released: the +// acting part, the step and the build it acted for. A wait that ended superseded, or of a step that is no acting part's (put, +// delivery, kept-build, backup), is not an act and keeps nothing. +func (ix *Indexer) HearWaitEnded(ctx context.Context, w meshwants.WaitEndedEvent) error { + if w.Part == "" || w.Module == "" || (w.How != "ran" && w.How != "released") { + return nil + } + switch w.Step { + case meshwants.WaitStepPut, meshwants.WaitStepDelivery, meshwants.WaitStepKeptBuild, meshwants.WaitStepBackup, "": + return nil + } + if ix.Heard == nil { + return errors.New("the state heard is not kept here") + } + a := Acted{V: 1, Part: w.Part, Step: w.Step, Build: w.Version, How: w.How, At: w.Since} + return ix.keepNewest(ctx, ActedKey(w.Part, w.Module), a, func(kept []byte) bool { + var k Acted + return json.Unmarshal(kept, &k) != nil || !a.At.Before(k.At) + }) +} + +// ActedKey is the key of the state heard of an acting part's act for a dependent. +func ActedKey(part, dependent string) string { + return ActedPrefix + keys.EscapeName(part) + "." + keys.EscapeName(dependent) +} + +// actedFor is the act kept for the acting part (an artifact reference, or its module) and the dependent module. +func (ix *Indexer) actedFor(ctx context.Context, part, dependent string) (Acted, bool) { + for _, p := range []string{part, strings.SplitN(part, ":", 2)[0]} { + raw, _, found, err := ix.Heard.Get(ctx, ActedKey(p, dependent)) + var a Acted + if err == nil && found && json.Unmarshal(raw, &a) == nil { + return a, true + } + } + return Acted{}, false +} + // keepNewest writes body at key of the state heard unless it holds a value whose version is higher; version of the body // is read by newer, which says whether the heard one is not older than the kept. func (ix *Indexer) keepNewest(ctx context.Context, key string, body any, newer func(kept []byte) bool) error { @@ -103,11 +156,14 @@ func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.Ve return d, true } -// Outdated answers the verb outdated (design 47 G §375) for the edge kinds 1, 2 and 4: every entry of each module's newest -// released version that is behind by the part's digest: kind 1, the recorded digest differs from the part's digest in the -// dependency's newest released version; kinds 2 and 4, from the dependency's newest version-delivered. A stale edge (kind 1) -// has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that left the digest as it -// was makes nothing behind. +// Outdated answers the verb outdated (design 47 G §375) for every edge kind: every entry of each module's newest released +// version that is behind. Kind 1: the recorded digest differs from the part's digest in the dependency's newest released +// version. Kind 3: the recorded toolchain version differs from the version of the newest toolchain-agreed of the dependent +// artifact's language (A-80, toolchain.). Kinds 2 and 4: from the dependency's newest version-delivered. Kind 5: the +// acting part has not acted for the dependent's current build (acted.. against the newest trunk +// entry's fingerprint, A-78 a). Kind 6: the delivered version's catalog entry no longer serves the verb or word (A-78 b). A +// stale edge (kinds 1, 3) has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that +// left the digest as it was makes nothing behind. func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (meshcatalog.OutdatedAnswer, error) { out := meshcatalog.OutdatedAnswer{Behind: []meshcatalog.Outdated{}} if ix.Versions == nil || ix.Heard == nil { @@ -136,6 +192,16 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( } } } + trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them + if ix.Revisions != nil { + if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found { + if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil { + for _, e := range entries { + trunk[e.Module] = e + } + } + } + } for _, m := range sortedKeys(newest) { v := newest[m] for _, e := range v.Edges { @@ -143,32 +209,62 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( if in.Dependency != "" && in.Dependency != e.Part { continue } - var latest, digest string + o := meshcatalog.Outdated{Module: m, Version: v.Version, Kind: e.Kind, + Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: ix.firstBranch(ctx, dep)}} switch e.Kind { case 1: n, ok := newest[dep] if !ok { continue } + var digest string for _, a := range n.Artifacts { if a.Artifact == e.Part { - latest, digest = n.Version, a.Digest + digest = a.Digest } } + if digest == "" || digest == e.Digest { + continue + } + o.Recorded, o.Newest = e.Version+" "+e.Digest, n.Version+" "+digest + case 3: + tc, ok := ix.agreedToolchain(ctx, languageOf(trunk[m], e)) + if !ok || tc.Version == "" || tc.Version == e.Version { + continue + } + o.Recorded, o.Newest = e.Version+" "+e.Digest, tc.Version case 2, 4: d, ok := ix.deliveredOf(ctx, dep) if !ok || e.Version == notYetDelivered { continue } - latest, digest = d.Version, d.Parts[e.Part] + digest := d.Parts[e.Part] + if digest == "" || digest == e.Digest { + continue + } + o.Recorded, o.Newest = e.Version+" "+e.Digest, d.Version+" "+digest + case 5: + cur := trunk[m].Fp + if cur == "" { + continue + } + acted, ok := ix.actedFor(ctx, e.Part, m) + if ok && acted.Build == cur { + continue + } + o.Recorded, o.Newest = "acted for "+cmp.Or(acted.Build, "no build"), "the current build "+cur + case 6: + d, ok := ix.deliveredOf(ctx, dep) + if !ok || e.Version == notYetDelivered || d.Version == e.Version { + continue + } + if serves, known := ix.deliveredServes(ctx, dep, d.Version, e.Word); !known || serves { + continue + } + o.Recorded, o.Newest = e.Version+" "+e.Word, d.Version+" no longer serves "+e.Word default: continue } - if digest == "" || digest == e.Digest { - continue - } - o := meshcatalog.Outdated{Module: m, Version: v.Version, Kind: e.Kind, Recorded: e.Version + " " + e.Digest, Newest: latest + " " + digest, - Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: ix.firstBranch(ctx, dep)}} if e.Stale { o.Act = "rebuild" } @@ -178,6 +274,88 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( return out, nil } +// agreedToolchain is the newest toolchain-agreed kept for a language (A-80). +func (ix *Indexer) agreedToolchain(ctx context.Context, language string) (meshbuilds.ToolchainAgreed, bool) { + var t meshbuilds.ToolchainAgreed + if language == "" { + return t, false + } + raw, _, found, err := ix.Heard.Get(ctx, ToolchainPrefix+keys.EscapeName(language)) + if err != nil || !found || json.Unmarshal(raw, &t) != nil { + return t, false + } + return t, true +} + +// languageOf is the language of the artifact a kind-3 edge of the entry builds: the target of the catalog graph's edge from the +// toolchain part, else the one language the entry's artifacts share. +func languageOf(entry meshcatalog.CatalogEntry, e meshcatalog.VersionEdge) string { + lang := map[string]string{} + for _, a := range entry.Artifacts { + lang[a.Ref] = a.Language + } + for _, g := range entry.Edges { + if g.Kind == 3 && (g.Part == e.Part || g.From == e.Part) && lang[g.To] != "" { + return lang[g.To] + } + } + one := "" + for _, l := range lang { + if l == "" || (one != "" && one != l) { + return "" + } + one = l + } + return one +} + +// deliveredServes says whether the catalog entry of a delivered version of module serves word, its verb (seat.verb, as the entry's +// serves lists it, the form `seat:` allowed) or a word of its manifest (A-78 b); known is false where that version's entry cannot be read. +func (ix *Indexer) deliveredServes(ctx context.Context, module, version, word string) (serves, known bool) { + if ix.Revisions == nil { + return false, false + } + vs, err := ModuleVersions(ctx, ix.Versions, module) + if err != nil { + return false, false + } + var source int64 + for _, v := range vs { + if v.Version == version { + source = v.Source + } + } + if source == 0 { + return false, false + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return false, false + } + var numbers []int64 + for _, n := range names { + if r, ok := TrunkRevisionOf(n); ok { + numbers = append(numbers, r) + } + } + slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) }) + for _, n := range numbers { // the newest revision taken from the source revision the version was built at + rev, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", n)) + if err != nil || rev.Source != source { + continue + } + for _, e := range entries { + if e.Module != module { + continue + } + w := strings.TrimPrefix(word, "seat:") + _, inManifest := e.Manifest[word] + return slices.Contains(e.Serves, w) || slices.Contains(e.Provides, w) || slices.Contains(e.Streams, w) || inManifest, true + } + } + return false, false +} + // firstBranch is the first watched branch of a module's registration, which its released versions are of. func (ix *Indexer) firstBranch(ctx context.Context, module string) string { if ix.Registry != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 75caf1f8..f35788f0 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -2,6 +2,7 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "slices" @@ -307,3 +308,106 @@ func TestWhatIsHeardIsKeptEscapedInTheStateHeard(t *testing.T) { t.Errorf("the own-build state holds %v", own) } } + +// Design 47 G §375, A-78, A-80: outdated says a kind-3 edge behind when the agreed toolchain of its artifact's language names another +// version, a kind-5 edge when the acting part has not acted for the dependent's current build (a wait-ended that ended ran or +// released), and a kind-6 edge when the delivered version's entry no longer serves the word. +func TestOutdatedAnswersKinds3And5And6(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)} + keep := func(n, source int64, serves []string, fp string) { + t.Helper() + entries := map[string]meshcatalog.CatalogEntry{ + "prov": {Module: "prov", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "prov:program"}}, Serves: serves}, + "actor": {Module: "actor", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}}, + "app": {Module: "app", Manifest: map[string]any{}, Fp: fp, Artifacts: []meshcatalog.Artifact{{Ref: "app:program", Language: "go"}}, + Edges: []meshcatalog.Edge{{Kind: 3, From: "tc:program", To: "app:program", Part: "tc:program"}, + {Kind: 5, From: "actor:program", To: "app:program", Part: "actor:program"}, + {Kind: 6, From: "prov:program", To: "app:program", Part: "prov:program", Word: "seat:s.verb"}}}, + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: n, Source: source, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + } + keep(1, 10, []string{"s.verb"}, "fp1") + put := func(module string, v meshcatalog.VersionEntry) { + t.Helper() + v.Module, v.V = module, 1 + data, _ := json.Marshal(v) + if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil { + t.Fatal(err) + } + } + put("prov", meshcatalog.VersionEntry{Version: "0.1.1", Source: 10}) + put("app", meshcatalog.VersionEntry{Version: "0.1.1", Edges: []meshcatalog.VersionEdge{ + {Kind: 3, Part: "tc:program", Version: "1.25", Digest: "t1", Stale: true}, + {Kind: 5, Part: "actor:program"}, + {Kind: 6, Part: "prov:program", Word: "seat:s.verb", Version: "0.1.1"}}}) + behind := func() map[int64]meshcatalog.Outdated { + t.Helper() + a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{}) + if err != nil { + t.Fatal(err) + } + m := map[int64]meshcatalog.Outdated{} + for _, o := range a.Behind { + m[o.Kind] = o + } + return m + } + if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil { + t.Fatal(err) + } + b := behind() + if _, ok := b[3]; ok { + t.Errorf("kind 3 behind at the agreed version: %+v", b[3]) + } + if o, ok := b[5]; !ok || o.Act != "" { + t.Errorf("kind 5 is not behind before the acting part acted: %+v", b) + } + wait := func(how, step, build string, since time.Time) { + t.Helper() + if err := ix.HearWaitEnded(ctx, meshwants.WaitEndedEvent{Part: "actor:program", Module: "app", Step: step, How: how, Version: build, Since: since}); err != nil { + t.Fatal(err) + } + } + at := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + wait("superseded", "compose", "fp1", at) + wait("ran", "delivery", "fp1", at) + if _, ok := behind()[5]; !ok { + t.Error("a superseded wait or a delivery wait counted as an act") + } + wait("ran", "compose", "fp1", at) + if _, ok := behind()[5]; ok { + t.Error("kind 5 is behind after the act for the current build") + } + keep(2, 20, []string{"s.other"}, "fp2") + if o, ok := behind()[5]; !ok || !strings.Contains(o.Newest, "fp2") { + t.Errorf("kind 5 after the dependent's new build: %+v", o) + } + wait("released", "compose", "fp2", at.Add(time.Minute)) + if _, ok := behind()[5]; ok { + t.Error("a released wait is not an act") + } + if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil { + t.Fatal(err) + } + if o, ok := behind()[3]; !ok || o.Act != "rebuild" || o.Newest != "1.26" || !strings.Contains(o.Recorded, "1.25") { + t.Errorf("kind 3 after the agreed toolchain changed: %+v", o) + } + // kind 6: prov 0.1.2 built at source 20 serves another word only + if _, ok := behind()[6]; ok { + t.Error("kind 6 behind with no newer delivered version") + } + put("prov", meshcatalog.VersionEntry{Version: "0.1.2", Source: 20}) + if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "prov", Version: "0.1.2", Parts: map[string]string{"prov:program": "p2"}}); err != nil { + t.Fatal(err) + } + if o, ok := behind()[6]; !ok || o.Act != "" || !strings.Contains(o.Newest, "no longer serves") { + t.Errorf("kind 6 after a version that serves no more: %+v", o) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 2ac33f79..e18f3fa8 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -161,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "mesh-wants.version-delivered", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index f9427b76..7daa9d0e 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -86,6 +86,7 @@ "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-wants.version-delivered", + "mesh-wants.wait-ended", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.source-moved", From bdc9a61524ae73a34148d1abfc7d7bac1c360d78 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:17:28 +0100 Subject: [PATCH 08/13] indexer: the first fill's edge entries, marked filled, from the placement and the stream running read whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VersionEdge gains filled (mesh-catalog seat contract, regenerated). Builds: design 47 G §377, A-78 c, A-62, §556 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- contracts/go/seats/meshcatalog/contract.go | 8 +- contracts/seats/mesh-catalog/seat.json | 4 + contracts/ts/seats/meshcatalog/index.d.ts | 2 + .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/versions.go | 57 +++++++ .../internal/index/candidates.go | 6 +- .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/versions_fill.go | 32 ++++ .../internal/index/versions_fill_edges.go | 159 ++++++++++++++++++ .../internal/index/versions_test.go | 72 ++++++++ .../mesh-delivery-indexer/manifest_test.go | 6 +- modules/mesh-delivery-indexer/module.json | 2 + 13 files changed, 344 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go diff --git a/contracts/go/seats/meshcatalog/contract.go b/contracts/go/seats/meshcatalog/contract.go index 34b65f16..ec2ccf76 100644 --- a/contracts/go/seats/meshcatalog/contract.go +++ b/contracts/go/seats/meshcatalog/contract.go @@ -1034,9 +1034,11 @@ type VersionEdge struct { // the digest of the part pointed to, kinds 1-4 Digest string `json:"digest,omitempty"` // kinds 5 and 6: the acting part, its step and the build that acted, or the verb or word used and the delivered version that serves it - Fact string `json:"fact,omitempty"` - Kind int64 `json:"kind"` - Part ArtifactRef `json:"part"` + Fact string `json:"fact,omitempty"` + // the entry was derived at the first fill (design 47 G section 377), not recorded at a release or a delivery + Filled bool `json:"filled,omitempty"` + Kind int64 `json:"kind"` + Part ArtifactRef `json:"part"` // the edge makes the dependent stale (kinds 1, 3) or only orders it (kinds 2, 4, 5, 6) (section 378) Stale bool `json:"stale"` // the dependency's version, kinds 1-4; "not yet delivered" until the first delivery for 2, 4 and 6 diff --git a/contracts/seats/mesh-catalog/seat.json b/contracts/seats/mesh-catalog/seat.json index 40fba93d..a233e12a 100644 --- a/contracts/seats/mesh-catalog/seat.json +++ b/contracts/seats/mesh-catalog/seat.json @@ -2555,6 +2555,10 @@ "fact": { "type": "string", "description": "kinds 5 and 6: the acting part, its step and the build that acted, or the verb or word used and the delivered version that serves it" + }, + "filled": { + "type": "boolean", + "description": "the entry was derived at the first fill (design 47 G section 377), not recorded at a release or a delivery" } }, "required": [ diff --git a/contracts/ts/seats/meshcatalog/index.d.ts b/contracts/ts/seats/meshcatalog/index.d.ts index dc83114e..dd3e2e5e 100644 --- a/contracts/ts/seats/meshcatalog/index.d.ts +++ b/contracts/ts/seats/meshcatalog/index.d.ts @@ -490,6 +490,8 @@ export interface VersionEdge { "digest"?: string; /** kinds 5 and 6: the acting part, its step and the build that acted, or the verb or word used and the delivered version that serves it */ "fact"?: string; + /** the entry was derived at the first fill (design 47 G section 377), not recorded at a release or a delivery */ + "filled"?: boolean; "kind": number; "part": ArtifactRef; /** the edge makes the dependent stale (kinds 1, 3) or only orders it (kinds 2, 4, 5, 6) (section 378) */ diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 8b9e0f39..69e3c1d4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -441,7 +441,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, + OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index 1f624d09..ce127695 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -10,6 +10,8 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" @@ -159,3 +161,58 @@ func afterRevision(say func(context.Context, meshcatalog.CatalogRevisionEvent) e return err } } + +// placementVia asks the seat mesh-module-assigner's verb placement for one module (A-62), a read. +func placementVia(call toolCaller) index.PlacementAsker { + return func(_ context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) { + raw, err := call(meshmoduleassigner.Seat+"."+meshmoduleassigner.VerbPlacement, map[string]any{"module": module}) + if err != nil { + return nil, fmt.Errorf("%s.%s: %w", meshmoduleassigner.Seat, meshmoduleassigner.VerbPlacement, err) + } + var a meshmoduleassigner.PlacementAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of placement is not a module's placements: %w", err) + } + return a.Placements, nil + } +} + +// runningWholeVia reads every subject's last message of the node-engine seat's stream running through an ordered consumer made for +// that one read (design 47 G §556), by node and module. +func runningWholeVia(js jetstream.JetStream) index.RunningReader { + return func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) { + stream, err := js.Stream(ctx, nodeengine.StreamRunning) + if err != nil { + return nil, fmt.Errorf("the stream %s cannot be read whole: %w", nodeengine.StreamRunning, err) + } + cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) + if err != nil { + return nil, fmt.Errorf("the ordered consumer for the whole read of %s cannot be made: %w", nodeengine.StreamRunning, err) + } + out := map[string]map[string]nodeengine.Running{} + for { + batch, err := cons.FetchNoWait(256) + if err != nil { + return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err) + } + n := 0 + for m := range batch.Messages() { + n++ + var r nodeengine.Running + if json.Unmarshal(m.Data(), &r) != nil || r.Node == "" || r.Module == "" { + continue + } + if out[r.Node] == nil { + out[r.Node] = map[string]nodeengine.Running{} + } + out[r.Node][r.Module] = r + } + if err := batch.Error(); err != nil { + return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err) + } + if n == 0 { + return out, nil + } + } + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 608f8c1e..ce1ae9e0 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -193,7 +193,11 @@ type Indexer struct { // Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since), // toolchain., delivered., pending.., acted.., each key part escaped as a name // (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled. - Heard Store + Heard Store + // Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the + // edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered". + Placement PlacementAsker + RunningWhole RunningReader judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) // Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a // pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none. diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index f2c65992..36826d8d 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -908,7 +908,7 @@ func (ix *Indexer) dry() *dryRun { // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, - OwnBuildState: nil, Heard: nil, underSeats: true, + OwnBuildState: nil, Heard: nil, Placement: nil, RunningWhole: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, // A dry run writes no registration and asks no one. diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 43a0003c..dccf55ff 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go index 4e4e8367..49d29256 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -5,9 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "maps" + "slices" "strconv" "time" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -127,6 +130,12 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { } now := ix.now().UTC() filled := 0 + built := map[string]meshcatalog.VersionEntry{} + claimed := map[string]time.Time{} + trunk := map[string]meshcatalog.CatalogEntry{} + for _, e := range entries { + trunk[e.Module] = e + } for module, fs := range best { v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: fs[0].v.Version, At: &now} for _, f := range fs { @@ -134,7 +143,29 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { if n, err := strconv.ParseInt(f.v.Revision, 10, 64); err == nil && n > v.Source { v.Source = n } + if f.v.At.After(claimed[module]) { + claimed[module] = f.v.At + } } + built[module] = v + } + // The edge entries (§377), each marked filled: where the stream running or the placement cannot be read the entries of kinds 2, 4 + // and 6 read "not yet delivered", as at a release, and the first delivery writes them. + var running map[string]map[string]nodeengine.Running + if ix.RunningWhole != nil { + if running, err = ix.RunningWhole(ctx); err != nil { + ix.logf("versions fill: the stream running is not read whole: %v", err) + running = nil + } + } + answers := map[string]BuilderVersionsAnswer{} + for _, module := range slices.Sorted(maps.Keys(built)) { + v := built[module] + v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, running, answers) + built[module] = v + } + for _, module := range slices.Sorted(maps.Keys(built)) { + v := built[module] data, err := json.Marshal(v) if err != nil { return err @@ -142,6 +173,7 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil && !errors.Is(err, ErrConflict) { return fmt.Errorf("the version %s %s is not kept: %w", module, v.Version, err) } + ix.keepFilledFacts(ctx, v, trunk[module], built, now) filled++ } if ix.SayVersionsFilled != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go new file mode 100644 index 00000000..a47075f6 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go @@ -0,0 +1,159 @@ +package index + +import ( + "cmp" + "context" + "encoding/json" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The edge entries of the first fill (design 47 G §377, A-78 c): each module's newest version gets its entries once, each marked +// filled. Kinds 1 and 3 come from the dependency version published latest before the version's build was claimed, read with +// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 from the version every machine placing the dependency +// runs, present or away, the machines from the assigner's placement (A-62) and each node's last running from the node-engine seat's +// stream read whole (§556); kind 5 as done for every acting part's running build. + +// PlacementAsker asks the seat mesh-module-assigner's verb placement for a module (a read, A-62). +type PlacementAsker func(ctx context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) + +// RunningReader reads the node-engine seat's stream running whole, every subject's last message (§556): by node, then module. +type RunningReader func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) + +// filledEdges are the edge entries of the entry's newest version, which was filled as built; claimed is when its build was claimed. +func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEntry, claimed time.Time, ask VersionsAsker, trunk map[string]meshcatalog.CatalogEntry, + built map[string]meshcatalog.VersionEntry, running map[string]map[string]nodeengine.Running, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge { + var out []meshcatalog.VersionEdge + seen := func(e meshcatalog.VersionEdge) bool { + return slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word }) + } + for _, g := range entry.Edges { + part := cmp.Or(g.Part, g.From) + module := strings.SplitN(part, ":", 2)[0] + if part == "" || module == entry.Module { + continue + } + e := meshcatalog.VersionEdge{Kind: g.Kind, Part: part, Word: g.Word, Filled: true} + switch g.Kind { + case 1, 3: + dep, ok := trunk[module] + if !ok { + continue + } + var name string + for _, a := range dep.Artifacts { + if a.Ref == part { + name = versionNameOf(module, a) + } + } + if name == "" { + continue + } + ans, ok := answers[name] + if !ok { + a, err := ask(ctx, name) + if err != nil { + continue + } + ans, answers[name] = a, a + } + var best *BuilderVersion + for i, v := range ans.Versions { + if _, ok := versionParts(v.Version); !ok || v.State != "published" || v.At.After(claimed) { + continue + } + if best == nil || v.At.After(best.At) { + best = &ans.Versions[i] + } + } + if best == nil { + continue + } + e.Version, e.Digest, e.Stale = best.Version, best.Digest, true + case 2, 4, 6: + e.Version = notYetDelivered + if v, ok := ix.runningVersion(ctx, module, trunk, built, running); ok { + e.Version = v.Version + for _, a := range v.Artifacts { + if a.Artifact == part { + e.Digest = a.Digest + } + } + } + case 5: + e.Fact = "acted for the running build " + entry.Fp + default: + continue + } + if !seen(e) { + out = append(out, e) + } + } + return out +} + +// runningVersion is the version of a module that every machine placing it runs, present or away, when each runs the newest +// build; false where no machine places it, a placed machine runs none or another build, or the placement is not answered. +func (ix *Indexer) runningVersion(ctx context.Context, module string, trunk map[string]meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, + running map[string]map[string]nodeengine.Running) (meshcatalog.VersionEntry, bool) { + v, ok := built[module] + e, known := trunk[module] + if !ok || !known || e.Fp == "" || ix.Placement == nil { + return v, false + } + placements, err := ix.Placement(ctx, module) + if err != nil { + ix.logf("versions fill: the placement of %s is not answered: %v", module, err) + return v, false + } + nodes := 0 + for _, p := range placements { + for _, n := range p.Nodes { + nodes++ + r, ok := running[n][module] + if !ok || r.Stopped || r.Fingerprint != e.Fp { + return v, false + } + } + } + return v, nodes > 0 +} + +// keepFilledFacts keeps what the first fill derived of a version as heard: for each entry of kinds 2, 4 and 6 read as delivered, +// the dependency's version as its newest delivered one (A-68), and for each kind-5 edge the act for the running build (A-78 a). +func (ix *Indexer) keepFilledFacts(ctx context.Context, v meshcatalog.VersionEntry, entry meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, at time.Time) { + for _, e := range v.Edges { + switch e.Kind { + case 2, 4, 6: + dep := built[strings.SplitN(e.Part, ":", 2)[0]] + if e.Version == notYetDelivered || dep.Version != e.Version { + continue + } + parts := map[string]string{} + for _, a := range dep.Artifacts { + parts[a.Artifact] = a.Digest + } + d := meshwants.VersionDeliveredEvent{V: 1, Module: dep.Module, Version: dep.Version, Parts: parts, At: &at} + if err := ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(dep.Module), d, func([]byte) bool { return false }); err != nil { + ix.logf("versions fill: the delivered version of %s is not kept: %v", dep.Module, err) + } + case 5: + data, _ := json.Marshal(Acted{V: 1, Part: e.Part, Build: entry.Fp, How: "filled", At: at}) + if err := write(ctx, ix.Heard, ActedKey(e.Part, v.Module), func(_ []byte, found bool) ([]byte, bool, error) { + if found { + return nil, false, nil + } + return data, false, nil + }); err != nil { + ix.logf("versions fill: the act for %s is not kept: %v", v.Module, err) + } + } + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index f35788f0..224982dc 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" @@ -411,3 +413,73 @@ func TestOutdatedAnswersKinds3And5And6(t *testing.T) { t.Errorf("kind 6 after a version that serves no more: %+v", o) } } + +// Design 47 G §377, A-78 c: the first fill gives each module's newest version its edge entries, each marked filled: kind 1 from the +// dependency version published latest before the build was claimed, kinds 2, 4 and 6 from the version every placing machine runs, +// read from the assigner's placement and the stream running, kind 5 as acted for the running build; so the first outdated says only what is truly behind. +func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) { + ctx := context.Background() + t0 := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + placed := map[string][]string{"prov": {"n1", "n2"}} + ix := &Indexer{Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 }, + Placement: func(_ context.Context, m string) ([]meshmoduleassigner.ModulePlacement, error) { + return []meshmoduleassigner.ModulePlacement{{Branch: "trunk", Nodes: placed[m]}}, nil + }, + RunningWhole: func(context.Context) (map[string]map[string]nodeengine.Running, error) { + return map[string]map[string]nodeengine.Running{"n1": {"prov": {Fingerprint: "fp-prov"}}, "n2": {"prov": {Fingerprint: "fp-prov"}}}, nil + }} + entries := map[string]meshcatalog.CatalogEntry{ + "lib": {Module: "lib", Manifest: map[string]any{}, Fp: "fp-lib", Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x/lib"}}}}, + "actor": {Module: "actor", Manifest: map[string]any{}, Fp: "fp-actor", Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}}, + "prov": {Module: "prov", Manifest: map[string]any{}, Fp: "fp-prov", Artifacts: []meshcatalog.Artifact{{Ref: "prov:program"}}}, + "app": {Module: "app", Manifest: map[string]any{}, Fp: "fp-app", Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}}, + Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true}, + {Kind: 4, From: "prov:program", To: "app:program", Part: "prov:program"}, + {Kind: 5, From: "actor:program", To: "app:program", Part: "actor:program"}}}, + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) { + pub := func(v, d string, at time.Time) BuilderVersion { + return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9", At: at} + } + switch name { + case "x/lib": + return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.1.1", "sha256:l1", t0), pub("0.1.2", "sha256:l2", t0.Add(2*time.Hour))}}, nil + case "app/program": + return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.3.1", "sha256:a1", t0.Add(time.Hour))}}, nil + } + return BuilderVersionsAnswer{Versions: []BuilderVersion{pub("0.9.1", "sha256:"+name, t0)}}, nil + } + if err := ix.FillVersions(ctx, ask); err != nil { + t.Fatal(err) + } + app, _ := ModuleVersions(ctx, ix.Versions, "app") + if len(app) != 1 || len(app[0].Edges) != 3 { + t.Fatalf("app's filled version: %+v", app) + } + for _, e := range app[0].Edges { + if !e.Filled { + t.Errorf("a filled entry is not marked: %+v", e) + } + switch e.Kind { + case 1: + if e.Version != "0.1.1" || e.Digest != "sha256:l1" || !e.Stale { + t.Errorf("kind 1 is the version published before the build was claimed: %+v", e) + } + case 4: + if e.Version != "0.9.1" || e.Digest != "sha256:prov/program" { + t.Errorf("kind 4 is the version every machine runs: %+v", e) + } + } + } + // lib's 0.1.2 was published after app's build was claimed: that is behind, and nothing else (the delivered kind 4 and the act of kind 5 are not) + if a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{}); err != nil || len(a.Behind) != 1 || a.Behind[0].Kind != 1 { + t.Errorf("outdated after the first fill: %+v, %v", a.Behind, err) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index e18f3fa8..c7c45db8 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -161,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -173,7 +173,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-module-assigner.placement", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle @@ -195,7 +195,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, - index.EntryIndexTool, // design 47 G §675 + index.EntryIndexTool, // design 47 G §675 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool) // design 47 G §667, §8 if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 7daa9d0e..dde0c85e 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -86,6 +86,7 @@ "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-wants.version-delivered", + "node-engine.running", "mesh-wants.wait-ended", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", @@ -157,6 +158,7 @@ "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", + "seat:mesh-module-assigner.placement", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", From 64def921ea54d682cda5299667e7ad3d2a5da837 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:19:29 +0100 Subject: [PATCH 09/13] indexer: an input is also keyed by its run ask's id and deleted at the build seat's step-ended of that ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeepInput takes the ask's id. It has no caller yet: the indexer publishes no run ask for its validation in this tree. Builds: design 47 G §354, A-78 d Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer/main.go | 4 ++ .../cmd/mesh-delivery-indexer/versions.go | 17 +++++ .../internal/index/run_inputs.go | 70 ++++++++++++++++--- .../internal/index/versions_test.go | 59 ++++++++++++++-- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + 6 files changed, 137 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 69e3c1d4..77aa3eee 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -25,6 +25,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -693,6 +694,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshwants.SubjectWaitEnded, func(m jetstream.Msg) { heardWaitEnded(ctx, ix, m, logger.Printf) }); err != nil { return err } + if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(ctx, ix, m, logger.Printf) }); err != nil { + return err + } if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index ce127695..1f0cdcd5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" @@ -112,6 +113,22 @@ func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log _ = m.Ack() } +// heardStepEnded deletes the input kept for the run ask a build seat's step-ended names (design 47 G §354, A-78 d). +func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var e nodebuildgo.StepEnded + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no step-ended, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.HearInputEnded(ctx, e); err != nil { + logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) + _ = m.Nak() + return + } + _ = m.Ack() +} + // sayVersionsFilled says versions-filled (design 47 G §372) once, on the seat's stream that keeps it with no age limit. func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.VersionsFilledEvent, string) error { return func(_ context.Context, e meshcatalog.VersionsFilledEvent, id string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/run_inputs.go b/modules/mesh-delivery-indexer/internal/index/run_inputs.go index 647737ae..ccb5070b 100644 --- a/modules/mesh-delivery-indexer/internal/index/run_inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/run_inputs.go @@ -9,10 +9,13 @@ import ( "errors" "fmt" "slices" + "strings" "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" ) // The indexer's inputs (design 47 G §354): the input of the ask `run` the indexer's validation puts, kept in the old state @@ -28,6 +31,9 @@ const ( // InputsSwept is how many of the oldest keys each new ask judges. InputsSwept = 20 inputsTime = "20060102T150405Z" + // InputsAskPrefix opens the keys that name an input by the id of the run ask it was kept for (A-78 d): ask., the id escaped as a + // name (§73), holding the input's own key, so the build seat's step-ended of that ask finds it and deletes it. + InputsAskPrefix = "ask." ) // InputDigest is the sha256 of an input's exact bytes, as "sha256:" and its hex digits. @@ -36,9 +42,13 @@ func InputDigest(body []byte) string { return "sha256:" + hex.EncodeToString(sum[:]) } -// KeepInput keeps the exact bytes of a run ask's input under their write time and sha256, after sweeping the keys more than a -// day old among the 20 oldest; it answers the digest. The same bytes written again in the same second are one key. -func (ix *Indexer) KeepInput(ctx context.Context, body []byte) (string, error) { +// KeepInput keeps the exact bytes of the input of the run ask ask under their write time and sha256, after sweeping the keys more +// than a day old among the 20 oldest, and names it by the ask's id (A-78 d); it answers the digest. The same bytes written again in +// the same second are one key. +func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (string, error) { + if ask == "" { + return "", errors.New("name the run ask the input is kept for") + } if ix.Inputs == nil { return "", errors.New("the state inputs is not kept here") } @@ -54,21 +64,53 @@ func (ix *Indexer) KeepInput(ctx context.Context, body []byte) (string, error) { if err := ix.Inputs.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) { return "", err } + if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) { + return "", err + } return digest, nil } +// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask +// that kept none, or whose input a sweep deleted, changes nothing. +func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error { + if ix.Inputs == nil || e.Ask == "" { + return nil + } + askKey := InputsAskPrefix + keys.EscapeName(e.Ask) + raw, rev, found, err := ix.Inputs.Get(ctx, askKey) + if err != nil || !found { + return err + } + if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil { + return err + } else if ok { + if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil { + return err + } + } + return ix.Inputs.Delete(ctx, askKey, rev) +} + // sweepInputs deletes, among the 20 oldest keys, those written more than a day ago. func (ix *Indexer) sweepInputs(ctx context.Context) error { all, err := ix.Inputs.Keys(ctx) if err != nil { return err } - slices.Sort(all) // the write time leads the key, so the oldest come first - if len(all) > InputsSwept { - all = all[:InputsSwept] - } var errs []error + var inputs, asks []string for _, k := range all { + if strings.HasPrefix(k, InputsAskPrefix) { + asks = append(asks, k) + } else { + inputs = append(inputs, k) + } + } + slices.Sort(inputs) // the write time leads the key, so the oldest come first + if len(inputs) > InputsSwept { + inputs = inputs[:InputsSwept] + } + for _, k := range inputs { t, _, ok := meshcatalog.ParseInputsByTimeDigestKey(k) at, perr := time.Parse(inputsTime, t) if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor { @@ -80,6 +122,16 @@ func (ix *Indexer) sweepInputs(ctx context.Context) error { } } } + // a name by ask whose input a sweep or its step-ended deleted goes with it + for _, k := range asks { + if raw, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found { + if _, _, ok, err := ix.Inputs.Get(ctx, string(raw)); err == nil && !ok { + if err := ix.Inputs.Delete(ctx, k, rev); err != nil { + errs = append(errs, err) + } + } + } + } return errors.Join(errs...) } @@ -90,7 +142,7 @@ func InputKey(ctx context.Context, st Store, digest string) (string, error) { return "", err } for _, k := range all { - if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest { + if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest && !strings.HasPrefix(k, InputsAskPrefix) { return k, nil } } @@ -162,7 +214,7 @@ func InputsReadTool(st Store) stdio.Tool { out := InputsAnswer{Inputs: []InputKept{}} for _, k := range all { _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k) - if !ok || (want != "" && d != want) { + if !ok || strings.HasPrefix(k, InputsAskPrefix) || (want != "" && d != want) { continue } raw, _, found, err := st.Get(ctx, k) diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 224982dc..8920edf1 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" ) // Design 47 G §349: a version is released once every artifact the module's entry names is published at it, said once as @@ -255,7 +256,7 @@ func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) { ctx := context.Background() now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }} - d1, err := ix.KeepInput(ctx, []byte("first")) + d1, err := ix.KeepInput(ctx, "ask-1", []byte("first")) if err != nil { t.Fatal(err) } @@ -267,18 +268,30 @@ func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) { t.Error("an input never kept is answered") } now = now.Add(23 * time.Hour) - if _, err := ix.KeepInput(ctx, []byte("second")); err != nil { + if _, err := ix.KeepInput(ctx, "ask-2", []byte("second")); err != nil { t.Fatal(err) } - if keys, _ := ix.Inputs.Keys(ctx); len(keys) != 2 { + inputsOnly := func() []string { + var out []string + all, _ := ix.Inputs.Keys(ctx) + for _, k := range all { + if !strings.HasPrefix(k, InputsAskPrefix) { + out = append(out, k) + } + } + return out + } + if keys := inputsOnly(); len(keys) != 2 { t.Fatalf("inside a day: %v", keys) } now = now.Add(2 * time.Hour) - d3, _ := ix.KeepInput(ctx, []byte("third")) - keys, _ := ix.Inputs.Keys(ctx) - if len(keys) != 2 { + d3, _ := ix.KeepInput(ctx, "ask-3", []byte("third")) + if keys := inputsOnly(); len(keys) != 2 { t.Fatalf("after a day the first stays: %v", keys) } + if all, _ := ix.Inputs.Keys(ctx); slices.Contains(all, InputsAskPrefix+"ask-1") { + t.Errorf("the name by ask of the swept input stays: %v", all) + } if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil { t.Error("the input a day old is still answered") } @@ -483,3 +496,37 @@ func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) { t.Errorf("outdated after the first fill: %+v, %v", a.Behind, err) } } + +// Design 47 G §354, A-78 d: an input is also keyed by its run ask's id, so the build seat's step-ended of that ask deletes it, and +// the key by ask no other read counts as an input. +func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) + ix := &Indexer{Inputs: newMem(), Now: func() time.Time { return now }} + d1, err := ix.KeepInput(ctx, "a.1", []byte("first")) + if err != nil { + t.Fatal(err) + } + d2, _ := ix.KeepInput(ctx, "a.2", []byte("second")) + if _, err := ix.KeepInput(ctx, "", []byte("x")); err == nil { + t.Error("an input kept for no ask") + } + if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "a.1"}); err != nil { + t.Fatal(err) + } + if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "a.1"}); err != nil { + t.Errorf("an end heard again: %v", err) + } + if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d1}); err == nil { + t.Error("the input of an ended ask is still answered") + } + if _, err := Input(ctx, ix.Inputs, meshcatalog.InputInput{Digest: d2}); err != nil { + t.Errorf("the input of another ask: %v", err) + } + if got, _ := ix.Inputs.Keys(ctx); len(got) != 2 { + t.Errorf("keys after one end: %v", got) + } + if a, err := InputsReadTool(ix.Inputs).Run(map[string]any{}); err != nil || len(a.(InputsAnswer).Inputs) != 1 { + t.Errorf("indexer_inputs counts a key by ask: %+v, %v", a, err) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index c7c45db8..37cd2b28 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -161,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index dde0c85e..2058d2cc 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -86,6 +86,7 @@ "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-wants.version-delivered", + "node-build-go.step-ended", "node-engine.running", "mesh-wants.wait-ended", "mesh-delivery-checker.verdict", From 13550328a65a017b3bf9e62a3eeeaafc1b2b6193 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:22:34 +0100 Subject: [PATCH 10/13] indexer: a catalog revision per watched branch, branch.. beside trunk., and revision answering per branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first watched branch's revisions stay trunk.; catalog-revision names its branch, escaped. Nothing yet takes a non-first branch's source revision in (KeepBranchEntries has no caller). Builds: design 47 G §291, §614, A-85 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/index/catalog_branches_test.go | 66 +++++++++++++++++ .../internal/index/catalog_revisions.go | 70 ++++++++++++++++--- .../internal/index/revision_verb.go | 26 ++++++- 3 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go new file mode 100644 index 00000000..f74724fd --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -0,0 +1,66 @@ +package index + +import ( + "context" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G A-85, §291: the indexer keeps a catalog revision per watched branch, trunk. for the first and +// branch.. (the branch escaped) for another, each numbered by its own branch, and the verb revision answers the +// branch it is asked for; the event names the branch, escaped. +func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Revisions: newMemObjects(1 << 20), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} + var said []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil } + reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Commit: "c1", Invokes: []string{RevisionInvoke}, Manifest: map[string]any{}} + tip := func(commit string) map[string]meshcatalog.CatalogEntry { + e := reader + e.Commit = commit + return map[string]meshcatalog.CatalogEntry{"reader": e} + } + if err := ix.KeepBranchEntries(ctx, "main", 5, tip("c1")); err != nil { + t.Fatal(err) + } + if err := ix.KeepBranchEntries(ctx, "feature/x.y", 7, tip("c2")); err != nil { + t.Fatal(err) + } + if err := ix.KeepBranchEntries(ctx, "feature/x.y", 8, tip("c3")); err != nil { + t.Fatal(err) + } + names, _ := ix.Revisions.Names(ctx) + have := map[string]bool{} + for _, n := range names { + have[n] = true + } + for _, want := range []string{"trunk.1", "branch.feature/x=2ey.1", "branch.feature/x=2ey.2"} { + if !have[want] { + t.Errorf("the objects hold %v, want %s", names, want) + } + } + if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 { + t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said) + } + answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { + a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision}) + if err != nil { + t.Fatalf("revision of %q: %v", branch, err) + } + return a + } + if a := answer("main", 0); a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { + t.Errorf("main answered %+v", a) + } + if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision) + } + if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's revision 1 answered %+v", a.Revision) + } + if a := answer("beta", 0); !a.NoneYet { + t.Errorf("a branch with no revision answered %+v, want none-yet", a) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index d27fa244..cea8ecec 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -14,6 +14,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -46,15 +47,41 @@ type Objects interface { // EntryObject is the object name of the entry of that digest. func EntryObject(digest string) string { return "entry." + digest } -// RevisionObject is the object name of a revision: trunk., or candidate.. for a candidate, +// TrunkBranch is the first watched branch, whose revisions are kept as trunk. (design 47 G §291, A-85). +const TrunkBranch = "main" + +// isTrunkBranch says whether a branch's revisions are the trunk's: the first watched branch, or none named. +func isTrunkBranch(branch string) bool { return branch == "" || branch == TrunkBranch } + +// BranchObject is the object name of a revision of a watched branch other than the first: branch.., +// the branch escaped as a name (A-85), so a dot in it never reads as a part. +func BranchObject(branch string, revision int64) string { + return "branch." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) +} + +// RevisionObject is the object name of a revision: trunk. for the first watched branch, +// branch.. for another watched branch (A-85), or candidate.. for a candidate, // which keeps its newest revision only. func RevisionObject(rev meshcatalog.CatalogRevision) string { if rev.Kind == "candidate" { return "candidate." + rev.Group + "." + rev.Heads } + if !isTrunkBranch(rev.Branch) { + return BranchObject(rev.Branch, rev.Revision) + } return "trunk." + strconv.FormatInt(rev.Revision, 10) } +// BranchRevisionOf is the revision number a branch object's name holds for that branch, and whether it is one. +func BranchRevisionOf(branch, name string) (int64, bool) { + n, ok := strings.CutPrefix(name, "branch."+keys.EscapeName(branch)+".") + if !ok { + return 0, false + } + r, err := strconv.ParseInt(n, 10, 64) + return r, err == nil +} + // TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. func TrunkRevisionOf(name string) (int64, bool) { n, ok := strings.CutPrefix(name, "trunk.") @@ -128,26 +155,37 @@ func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.Cata // newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false. func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) { + return newestOf(ctx, o, "") +} + +// newestOf is the newest revision of a watched branch the bucket keeps (the trunk's for the first watched branch or none +// named), its list only; none is a zero revision and false. +func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, bool, error) { names, err := o.Names(ctx) if err != nil { return meshcatalog.CatalogRevision{}, false, err } + number := TrunkRevisionOf + if !isTrunkBranch(branch) { + number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) } + } var newest int64 for _, n := range names { - if r, ok := TrunkRevisionOf(n); ok && r > newest { + if r, ok := number(n); ok && r > newest { newest = r } } if newest == 0 { return meshcatalog.CatalogRevision{}, false, nil } - data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10)) + name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest}) + data, err := o.Get(ctx, name) if err != nil { return meshcatalog.CatalogRevision{}, false, err } var rev meshcatalog.CatalogRevision if err := json.Unmarshal(data, &rev); err != nil { - return rev, false, fmt.Errorf("trunk.%d: %w", newest, err) + return rev, false, fmt.Errorf("%s: %w", name, err) } return rev, true, nil } @@ -246,12 +284,26 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal if ix.catchingUp.Load() { return nil // the catch-up of a first start keeps its one revision at its end (CatchUp) } - ix.keepMu.Lock() - defer ix.keepMu.Unlock() ix.mu.Lock() src := int64(ix.source.rev) ix.mu.Unlock() - newest, any, err := newestTrunk(ctx, ix.Revisions) + return ix.keepBranchEntries(ctx, TrunkBranch, src, entries) +} + +// KeepBranchEntries keeps entries as the revision of a watched branch after that branch's newest (design 47 G §291, +// A-85): trunk. for the first watched branch, branch.. for another, numbered per branch, and +// says catalog-revision naming the branch. src is the branch's source revision the entries were taken in from. +func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error { + return ix.keepBranchEntries(ctx, branch, src, entries) +} + +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error { + if branch == "" { + branch = TrunkBranch + } + ix.keepMu.Lock() + defer ix.keepMu.Unlock() + newest, any, err := newestOf(ctx, ix.Revisions, branch) if errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) return nil @@ -261,7 +313,7 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal return fmt.Errorf("the newest catalog graph revision: %w", err) } at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) if err != nil { return err } @@ -292,7 +344,7 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, - Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) } } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index ffcf0d07..d2ecc045 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -5,11 +5,11 @@ import ( "encoding/json" "fmt" "slices" - "strconv" "strings" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -53,6 +53,22 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke) } + // A watched branch other than the first keeps its own revisions (A-85); the first's are the trunk's. The branch is + // named as a key name is escaped, so it is read back to the branch's own name. + branch, err := keys.UnescapeName(in.Branch) + if err != nil && in.Branch != "" { + return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err) + } + if !isTrunkBranch(branch) { + other, found, err := newestOf(ctx, o, branch) + if err != nil { + return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err) + } + if !found { + return meshcatalog.RevisionAnswer{NoneYet: true}, nil + } + newest = other + } name := RevisionObject(newest) switch { case in.Group != "" || in.Heads != "": @@ -61,10 +77,14 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata } name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads}) case in.Revision != 0 && in.Revision != newest.Revision: - name = "trunk." + strconv.FormatInt(in.Revision, 10) + name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: in.Revision}) } rev, entries := newest, trunkEntries - if name != RevisionObject(newest) { + if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" { + if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + return out, err + } + } else if name != RevisionObject(newest) { if rev, entries, err = ReadRevision(ctx, o, name); err != nil { return out, err } From 31792235e61bc278d280aa4eafe0edfb8ae116b5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:24:41 +0100 Subject: [PATCH 11/13] indexer: the build-edges state for builds of another watched branch, pruned on registration-changed, outdated kind 1 against the branch's newest build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BuildEdges.fp is the entry's fingerprint in the branch's revision whose source the build names; the branch's newest build is kept as heard in heard as branch-build... Builds: design 47 G §374, §375, A-83, A-80, A-85 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer/main.go | 2 + .../internal/index/branch_builds.go | 243 ++++++++++++++++++ .../internal/index/branch_builds_test.go | 90 +++++++ .../internal/index/candidates.go | 3 + .../internal/index/pending_test.go | 2 +- .../internal/index/registrations.go | 8 + .../internal/index/versions.go | 4 +- .../internal/index/versions_edges.go | 5 + .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + 10 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/branch_builds.go create mode 100644 modules/mesh-delivery-indexer/internal/index/branch_builds_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 77aa3eee..2bddf629 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -450,6 +450,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro SayFill: sayFill(conn, cred), // The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released. Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)}, + BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.BuildEdgesBucket, "no build of another watched branch keeps its edges", logger.Printf)}, Inputs: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)}, SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; @@ -462,6 +463,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.RegistrationsBucket, "no registration is kept or written", logger.Printf)}, awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)}, routerRecordUnread, ix, logger.Printf) + registry.Pruned = ix.PruneBranchBuilds ix.Registry = registry // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKept), logf: logger.Printf} diff --git a/modules/mesh-delivery-indexer/internal/index/branch_builds.go b/modules/mesh-delivery-indexer/internal/index/branch_builds.go new file mode 100644 index 00000000..e72cb362 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/branch_builds.go @@ -0,0 +1,243 @@ +package index + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The builds of a watched branch other than the first release no version (design 47 G §371); the indexer keeps each one's +// edge entries in the state build-edges, under ., and what the branch's newest build of the module +// published as heard, branch-build.. in the state heard (§374, A-83). +const ( + BuildEdgesState = "build-edges" + BuildEdgesBucket = Module + "_" + BuildEdgesState + // BranchBuildPrefix opens branch-build.., each part escaped as a name (A-80). + BranchBuildPrefix = "branch-build." +) + +// BranchBuild is what a branch-build key holds: the newest build of a module on a watched branch that was heard published, +// its source revision, the fingerprint of the branch's catalog entry at that revision and each artifact's digest. +type BranchBuild struct { + V int64 `json:"v"` + Version string `json:"version"` + Source int64 `json:"source"` + Fp string `json:"fp"` + Artifacts map[string]string `json:"artifacts"` + At time.Time `json:"at"` +} + +// BranchBuildKey is the key of the state heard of a module's newest build on a branch. +func BranchBuildKey(module, branch string) string { + return BranchBuildPrefix + keys.EscapeName(module) + "." + keys.EscapeName(branch) +} + +// BuildEdgesKey is the key of the state build-edges of a module's build on a branch, each part escaped as a name. +func BuildEdgesKey(module, branch string) string { + return meshcatalog.BuildEdgesByModuleBranchKey(keys.EscapeName(module), keys.EscapeName(branch)) +} + +// branchEntryAt is the module's entry in the newest revision of the branch taken in from that source revision (A-83: the +// revision whose source the build names, which the indexer holds itself). +func (ix *Indexer) branchEntryAt(ctx context.Context, module, branch string, source int64) (meshcatalog.CatalogEntry, error) { + if ix.Revisions == nil { + return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry says the build's fingerprint") + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return meshcatalog.CatalogEntry{}, err + } + number := TrunkRevisionOf + if !isTrunkBranch(branch) { + number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) } + } + var numbers []int64 + for _, n := range names { + if r, ok := number(n); ok { + numbers = append(numbers, r) + } + } + slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) }) + for _, n := range numbers { + name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: n}) + rev, entries, err := ReadRevision(ctx, ix.Revisions, name) + if err != nil { + return meshcatalog.CatalogEntry{}, err + } + if rev.Source != source { + continue + } + for _, e := range entries { + if e.Module == module { + return e, nil + } + } + } + return meshcatalog.CatalogEntry{}, fmt.Errorf("the indexer keeps no revision of the branch %s at source revision %d with an entry of %s", branch, source, module) +} + +// hearBranchBuild takes the package-published of a build of a watched branch other than the first (A-83): the build is kept as +// heard, and its edge entries are kept in build-edges, keyed by the fingerprint of the entry in the revision of that branch the +// build's source names. A build of an older source revision than the one kept leaves what is kept. +func (ix *Indexer) hearBranchBuild(ctx context.Context, p meshbuilds.PackagePublished) error { + if ix.BuildEdges == nil { + return errors.New("the state build-edges is not kept here") + } + entry, err := ix.branchEntryAt(ctx, p.Module, p.Branch, p.Source) + if err != nil { + return err + } + if entry.Fp == "" { + ix.logf("build-edges: the entry of %s on %s at source revision %d has no fingerprint, so its build keeps none", p.Module, p.Branch, p.Source) + return nil + } + at := ix.now().UTC() + err = write(ctx, ix.Heard, BranchBuildKey(p.Module, p.Branch), func(raw []byte, found bool) ([]byte, bool, error) { + var b BranchBuild + if found && json.Unmarshal(raw, &b) == nil && b.Source > p.Source { + return nil, false, nil + } + if !found || b.Source != p.Source || b.Fp != entry.Fp { + b = BranchBuild{Artifacts: map[string]string{}} + } + b.V, b.Version, b.Source, b.Fp, b.At = 1, p.Version, p.Source, entry.Fp, at + b.Artifacts[p.Artifact] = p.Digest + data, err := json.Marshal(b) + return data, false, err + }) + if err != nil { + return err + } + edges := releaseEdges(entry, p.Dependencies) + return write(ctx, ix.BuildEdges, BuildEdgesKey(p.Module, p.Branch), func(raw []byte, found bool) ([]byte, bool, error) { + be := meshcatalog.BuildEdges{V: 1, Fp: entry.Fp, Source: p.Source} + if found { + kept, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw) + if err != nil { + return nil, false, err + } + if kept.Source > p.Source { + return nil, false, nil + } + if kept.Source == p.Source && kept.Fp == entry.Fp { + be.Edges = kept.Edges + } + } + for _, e := range edges { + if !slices.ContainsFunc(be.Edges, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word }) { + be.Edges = append(be.Edges, e) + } + } + data, err := json.Marshal(be) + return data, false, err + }) +} + +// PruneBranchBuilds forgets a module's builds of the branches its registration no longer watches beside its first +// (registration-changed, §374): its build-edges keys and its branch-build keys of every other branch. +func (ix *Indexer) PruneBranchBuilds(ctx context.Context, module string, watched []string) error { + keep := map[string]bool{} + if len(watched) > 1 { + for _, b := range watched[1:] { + keep[keys.EscapeName(b)] = true + } + } + var errs []error + forget := func(st Store, key string) { + if _, rev, found, err := st.Get(ctx, key); err != nil { + errs = append(errs, err) + } else if found { + if err := st.Delete(ctx, key, rev); err != nil && !errors.Is(err, ErrConflict) { + errs = append(errs, err) + } + } + } + if ix.BuildEdges != nil { + all, err := ix.BuildEdges.Keys(ctx) + if err != nil { + return err + } + for _, k := range all { + if m, b, ok := meshcatalog.ParseBuildEdgesByModuleBranchKey(k); ok && m == keys.EscapeName(module) && !keep[b] { + forget(ix.BuildEdges, k) + } + } + } + if ix.Heard != nil { + all, err := ix.Heard.Keys(ctx) + if err != nil { + return err + } + prefix := BranchBuildPrefix + keys.EscapeName(module) + "." + for _, k := range all { + if b, ok := strings.CutPrefix(k, prefix); ok && !keep[b] { + forget(ix.Heard, k) + } + } + } + return errors.Join(errs...) +} + +// outdatedBranchBuilds are the behind entries of kind 1 of the builds of other watched branches (A-83): the digest a build +// recorded for a package against the digest in the newest build of that package's module on the same branch, as heard, never a +// released digest. +func (ix *Indexer) outdatedBranchBuilds(ctx context.Context, in meshcatalog.OutdatedInput) ([]meshcatalog.Outdated, error) { + if ix.BuildEdges == nil || ix.Heard == nil { + return nil, nil + } + all, err := ix.BuildEdges.Keys(ctx) + if err != nil { + return nil, err + } + slices.Sort(all) + var out []meshcatalog.Outdated + for _, k := range all { + em, eb, ok := meshcatalog.ParseBuildEdgesByModuleBranchKey(k) + if !ok { + continue + } + module, err1 := keys.UnescapeName(em) + branch, err2 := keys.UnescapeName(eb) + if err1 != nil || err2 != nil || (in.Branch != "" && in.Branch != eb) { + continue + } + raw, _, found, err := ix.BuildEdges.Get(ctx, k) + if err != nil || !found { + continue + } + be, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw) + if err != nil { + return nil, fmt.Errorf("the key %s of the state %s: %w", k, BuildEdgesState, err) + } + for _, e := range be.Edges { + if e.Kind != 1 || (in.Dependency != "" && in.Dependency != e.Part) { + continue + } + var dep BranchBuild + raw, _, found, err := ix.Heard.Get(ctx, BranchBuildKey(strings.SplitN(e.Part, ":", 2)[0], branch)) + if err != nil || !found || json.Unmarshal(raw, &dep) != nil { + continue + } + digest := dep.Artifacts[e.Part] + if digest == "" || digest == e.Digest { + continue + } + o := meshcatalog.Outdated{Module: module, Build: be.Fp, Kind: 1, Recorded: e.Version + " " + e.Digest, Newest: dep.Version + " " + digest, + Dependency: meshcatalog.OutdatedDependency{Artifact: e.Part, Branch: branch}} + if e.Stale { + o.Act = "rebuild" + } + out = append(out, o) + } + } + return out, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go b/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go new file mode 100644 index 00000000..7741d14e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/branch_builds_test.go @@ -0,0 +1,90 @@ +package index + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G §374, A-83, A-85: a build of a watched branch other than the first releases no version; its edge entries are kept +// in build-edges under ., the fingerprint being the entry's in the branch's revision whose source the build +// names; outdated compares kind 1 with the same branch's newest build, as heard; registration-changed prunes the branch's +// keys when it is no longer watched. +func TestABuildOfAnotherWatchedBranchKeepsItsEdgesAndIsOutdatedAgainstThatBranch(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Versions: newMem(), Heard: newMem(), BuildEdges: newMem(), Revisions: newMemObjects(1 << 20), Registry: &Registry{Registrations: MemoryStore()}} + for _, m := range []string{"lib", "app"} { + raw, _ := json.Marshal(meshcatalog.Registration{Module: m, Branches: []string{"main", "beta"}}) + if err := ix.Registry.Registrations.Create(ctx, RegistrationKey(m), raw); err != nil { + t.Fatal(err) + } + } + fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) } + keep := func(source int64, libFp, appFp string) { + t.Helper() + err := ix.KeepBranchEntries(ctx, "beta", source, map[string]meshcatalog.CatalogEntry{ + "lib": {Module: "lib", Fp: libFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg"}}}, + "app": {Module: "app", Fp: appFp, Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "app:program"}}, + Edges: []meshcatalog.Edge{{Kind: 1, From: "lib:pkg", To: "app:program", Part: "lib:pkg", Stale: true, Word: "imports"}}}}) + if err != nil { + t.Fatal(err) + } + } + keep(7, fp("a"), fp("b")) + keep(8, fp("c"), fp("b")) + pub := func(module, artifact, digest string, source int64, deps ...meshbuilds.Dependency) { + t.Helper() + if err := ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: module, Artifact: artifact, Digest: digest, Version: "0.1.1", Source: source, Branch: "beta", Dependencies: deps}); err != nil { + t.Fatal(err) + } + } + pub("lib", "lib:pkg", "d1", 7) + pub("app", "app:program", "a1", 7, meshbuilds.Dependency{Kind: 1, Part: "lib:pkg", Version: "0.1.1", Digest: "d1"}) + if v, _ := ModuleVersions(ctx, ix.Versions, "app"); len(v) != 0 { + t.Errorf("a build of beta released %v", v) + } + raw, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")) + be, err := meshcatalog.DecodeBuildEdgesByModuleBranchValue(raw) + if !found || err != nil || be.Fp != fp("b") || be.Source != 7 || len(be.Edges) != 1 || be.Edges[0].Digest != "d1" { + t.Fatalf("build-edges of app on beta: %+v (found %v, %v), want the fingerprint of the entry at source 7 and its kind 1 edge", be, found, err) + } + behind := func() []meshcatalog.Outdated { + t.Helper() + a, err := ix.Outdated(ctx, meshcatalog.OutdatedInput{}) + if err != nil { + t.Fatal(err) + } + return a.Behind + } + if b := behind(); len(b) != 0 { + t.Fatalf("behind before lib's next build: %+v", b) + } + pub("lib", "lib:pkg", "d2", 8) + b := behind() + if len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Build != fp("b") || b[0].Version != "" || b[0].Dependency.Branch != "beta" || + !strings.Contains(b[0].Recorded, "d1") || !strings.Contains(b[0].Newest, "d2") || b[0].Act != "rebuild" { + t.Fatalf("behind after lib's newer build on beta: %+v", b) + } + if err := ix.PruneBranchBuilds(ctx, "app", []string{"main", "beta"}); err != nil { + t.Fatal(err) + } + if _, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")); !found { + t.Errorf("a branch still watched lost its build-edges") + } + if err := ix.PruneBranchBuilds(ctx, "app", []string{"main"}); err != nil { + t.Fatal(err) + } + if _, _, found, _ := ix.BuildEdges.Get(ctx, BuildEdgesKey("app", "beta")); found { + t.Errorf("build-edges of app on beta stayed after beta was unwatched") + } + if _, _, found, _ := ix.Heard.Get(ctx, BranchBuildKey("app", "beta")); found { + t.Errorf("branch-build of app on beta stayed after beta was unwatched") + } + if _, _, found, _ := ix.Heard.Get(ctx, BranchBuildKey("lib", "beta")); !found { + t.Errorf("lib's branch-build went with app's") + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index ce1ae9e0..a5d9a1bb 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -109,6 +109,9 @@ type Indexer struct { SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error // Versions is the state versions (versions.go, design 47 G §349): each module's released versions; nil keeps none. Versions Store + // BuildEdges is the state build-edges (branch_builds.go, design 47 G §374): the edge entries of the newest build of a module on a + // watched branch other than the first; nil keeps none. + BuildEdges Store // SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing. SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error // SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing. diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index dccf55ff..1b6b927b 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/registrations.go b/modules/mesh-delivery-indexer/internal/index/registrations.go index 3c0f3eae..4f776c26 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations.go @@ -121,6 +121,9 @@ type Registry struct { HistoryOf func(ctx context.Context, module string) ([]meshcatalog.Registration, error) // Say says the seat mesh-catalog's event registration-changed. Say func(ctx context.Context, e meshcatalog.RegistrationChangedEvent) error + // Pruned is told after a registration changed which branches the module watches, so what is kept of its builds of a branch + // it no longer watches goes (registration-changed, design 47 G §374); nil prunes none. + Pruned func(ctx context.Context, module string, watched []string) error // Branch reads a branch through the seat git; an empty branch asks the repository's default branch. Branch func(repository, branch string) (gitseat.BranchAnswer, error) // Ask publishes an ask under the indexer's own name, Cancel takes it back, Where reads the router's own record of an @@ -796,6 +799,11 @@ func (r *Registry) write(ctx context.Context, key string, p Proposal, next meshc r.logf("registration-changed of %s %d is not said: %v", next.Module, next.Version, err) } } + if r.Pruned != nil { + if err := r.Pruned(ctx, next.Module, next.Branches); err != nil { + r.logf("the builds of %s on the branches it no longer watches are not pruned: %v", next.Module, err) + } + } return r.endKey(ctx, key, p, ProposalApplied, fmt.Sprintf("applied as version %d of %s on %s", next.Version, next.Module, occ.Says(w))) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 8327575a..e0596ceb 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -100,7 +100,7 @@ func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog // HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its // version, and once every artifact the module's catalog entry names is published at it the version is released: recorded, // and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A -// build of a watched branch other than the registration's first releases none (§371). +// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83). func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error { if ix.Versions == nil || ix.Heard == nil { return errors.New("the state versions is not kept here") @@ -110,7 +110,7 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag } if ix.Registry != nil { if r, found, err := ix.Registry.Registration(ctx, p.Module); err == nil && found && len(r.Branches) > 0 && r.Branches[0] != p.Branch { - return nil + return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83) } } if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil || found { diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index d99a61f8..7691a6e6 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -271,6 +271,11 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( out.Behind = append(out.Behind, o) } } + behind, err := ix.outdatedBranchBuilds(ctx, in) + if err != nil { + return out, err + } + out.Behind = append(out.Behind, behind...) return out, nil } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 37cd2b28..80f4a6f9 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -125,7 +125,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, - {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.InputsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 + {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 2058d2cc..ed7f9fe6 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -119,6 +119,7 @@ { "name": "registrations", "history": 1 }, { "name": "proposals", "history": 1, "max-mib": 4 }, { "name": "versions", "history": 1, "max-mib": 64 }, + { "name": "build-edges", "history": 1 }, { "name": "inputs", "history": 1 } ], "streams": [ From e27160571a8efd0637a4133a250a4c8ada4e108c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:24:51 +0100 Subject: [PATCH 12/13] indexer: go.mod requires the nodeengine seat directly, which its code imports Builds: design 47 G A-86 (handover item 3: go mod tidy of the nodeengine require; tidy cannot run offline, the one line is made by hand) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-indexer/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index af3efc9b..91fed057 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -68,7 +68,7 @@ require ( ) require ( - git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 From b3cd0ef661ccc9f9bfb95759df35d16a86305ff8 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:31:30 +0100 Subject: [PATCH 13/13] indexer: a watched branch other than the first is taken in from its source into its own revision, a pure computation from its heads' merged messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sourceFollow takes source. for every branch the seat lists (the answer with no branch), ApplyBranchSource computes each module's fingerprint, inputs and graph node over a view in memory seeded from the branch's newest revision, and keeps it with KeepBranchEntries; the trunk's modules, manifests, taken state, graph and packages are not written. The trunk branch is named once (index.TrunkBranch). Builds: design 47 G §13, §14, §291, ADR 0666 §6, A-85, A-90, A-93 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-indexer/integration_test.go | 71 ++++++++++++++++++ .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/main_test.go | 13 +++- .../cmd/mesh-delivery-indexer/named.go | 2 +- .../cmd/mesh-delivery-indexer/source.go | 42 +++++++++-- .../internal/index/branch_take_in.go | 55 ++++++++++++++ .../internal/index/branch_take_in_test.go | 75 +++++++++++++++++++ .../internal/index/catalog_revisions.go | 24 +++++- 8 files changed, 268 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/branch_take_in.go create mode 100644 modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index ef756cfb..0e8a5a9f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -1414,3 +1414,74 @@ func serveAssignerAssignments(t *testing.T, ctx context.Context, js jetstream.Je } t.Cleanup(func() { _ = sub.Unsubscribe() }) } + +// TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision: the seat's verb source lists a second watched branch, +// beta, with the merge of mesh-catalog it names as its head (design 47 G §13, §14, §291, A-93); the indexer started with the +// branch listed reads source.beta, takes the head's merged message in as a pure computation, and keeps branch.beta.1, which +// the branch follower (index.BranchView, as the branch's builds read it) reads; the trunk's modules are left. +func TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision(t *testing.T) { + manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") + if err != nil { + t.Fatal(err) + } + const merge = "89abcdef0123456789abcdef0123456789abcdef" + data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "beta", Number: 301, + Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", + Trees: map[string]string{"modules/avahi": "betatree"}, Manifests: map[string]string{"modules/avahi": string(manifest)}, + Locks: map[string]map[string]string{"modules/avahi": {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}}) + if err != nil { + t.Fatal(err) + } + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { + builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent") + }, func(ctx context.Context, js jetstream.JetStream) { + seedBuildModules(t, ctx, js, nil) + source := func(m *nats.Msg) { + var ask struct{ Branch string } + _ = json.Unmarshal(m.Data, &ask) + answer := meshdelivery.SourceAnswer{Branch: ask.Branch} + switch ask.Branch { + case "": + answer.Branches = []string{"main", "beta"} + case "beta": + answer.Group, answer.Revision = "g", 9 + answer.Repositories = []meshdelivery.SourceHead{{Repository: "novox/mesh-catalog", Head: merge, Trunk: "beta"}} + } + raw, _ := json.Marshal(map[string]any{"result": answer, "node": "n1"}) + _ = m.Respond(raw) + } + bridge := func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data)}, "node": "n1"}) + _ = m.Respond(raw) + } + for subject, h := range map[string]nats.MsgHandler{ + "mesh.seat.mesh-delivery.call.source.>": source, "mesh.seat.mesh-delivery.tool.source": source, + "mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>": bridge, "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message": bridge, + } { + if _, err := js.Conn().Subscribe(subject, h); err != nil { + t.Fatal(err) + } + } + }) + objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects) + var view index.Store + var at int64 + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && at == 0; time.Sleep(50 * time.Millisecond) { + if v, _, n, err := index.BranchView(ctx, objects, "beta"); err == nil { + view, at = v, n + } + } + if at != 1 { + t.Fatalf("the branch follower reads beta at revision %d within 20 s of the start, want 1", at) + } + value, _, found, err := view.Get(ctx, "avahi") + if err != nil || !found { + t.Fatalf("branch.beta.1 holds no avahi (%v)", err) + } + if m, err := meshcatalogclient.DecodeModule(value); err != nil || m.Commit != merge || m.Inputs2 == nil { + t.Errorf("avahi at the branch's head = %+v (%v), want commit %s with its inputs", m, err, merge) + } + if e, err := modules.Get(ctx, "avahi"); err == nil { + t.Errorf("modules.avahi = %s, want the trunk's modules left", e.Value()) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 2bddf629..eb3047e3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -511,7 +511,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a // kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard // meanwhile is read after. - source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, logf: logger.Printf} + source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf} go source.catchUp(ctx) // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the // assignments is settled first, so the toolchains settle through the gate already knowing it. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index dd54b4e6..fa510722 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -191,6 +191,9 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt( s = &sourceFollow{ read: func(branch string) (meshdelivery.SourceAnswer, error) { order = append(order, "read "+branch) + if branch == "" { // given no branch the seat lists the branches it keeps a source of (A-93) + return meshdelivery.SourceAnswer{Branches: []string{"main", "beta", "feat=2e1"}}, nil + } if revision == 0 { return meshdelivery.SourceAnswer{}, fmt.Errorf("source.main is not readable") } @@ -205,15 +208,21 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt( order = append(order, fmt.Sprintf("apply %s %d", a.Branch, a.Revision)) return nil }, + applyBranch: func(_ context.Context, branch string, a meshdelivery.SourceAnswer) error { + order = append(order, fmt.Sprintf("branch %s %d", branch, a.Revision)) + return nil + }, logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }, } s.catchUp(context.Background()) - want := []string{"read main", "apply main 4", "read main", "apply main 4"} // once, after the catch-up; beta is left + // The trunk, then each other watched branch the seat lists (a name unescaped), then what was heard meanwhile once each, + // beta among them: main through the take-in, every other branch into its own revision (A-93). + want := []string{"read main", "apply main 4", "read ", "read beta", "branch beta 4", "read feat=2e1", "branch feat.1 4", "read main", "apply main 4", "read beta", "branch beta 4"} if !reflect.DeepEqual(order, want) { t.Errorf("order = %v, want %v", order, want) } s.moved(context.Background(), "main") // after the catch-up: read at once - if len(order) != 6 { + if len(order) != 13 { t.Errorf("a source-moved after the catch-up was not read at once: %v", order) } order, revision = nil, 0 diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index cc1f6db7..a08fda8c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -237,7 +237,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str } // trunkBranch is the branch pull requests are open into. -const trunkBranch = "main" +const trunkBranch = index.TrunkBranch // openVia asks the forge which pull requests into the trunk are open now (the seat git's open-pulls), every repository's. // An answer that lacks the list is refused, never read as none. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go index 10f92615..16b62e1f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go @@ -8,6 +8,7 @@ import ( "sync" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" "github.com/nats-io/nats.go/jetstream" ) @@ -40,17 +41,31 @@ func sourceVia(call toolCaller) func(branch string) (meshdelivery.SourceAnswer, type sourceFollow struct { read func(branch string) (meshdelivery.SourceAnswer, error) apply func(context.Context, meshdelivery.SourceAnswer) error - logf func(string, ...any) + // applyBranch takes a watched branch other than the trunk's in (design 47 G §291, A-93); branch is its name, unescaped. + applyBranch func(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error + logf func(string, ...any) mu sync.Mutex caught bool waiting map[string]bool } -// catchUp reads the trunk branch's source revision and applies it, then the branches a source-moved named meanwhile. A -// read that fails is said; the catch-up ends all the same, and the next source-moved reads again. +// catchUp reads the trunk branch's source revision and applies it, then every other branch the seat keeps a source of (the +// answer given no branch lists them, A-93), then the branches a source-moved named meanwhile. A read that fails is said; +// the catch-up ends all the same, and the next source-moved reads again. func (s *sourceFollow) catchUp(ctx context.Context) { s.take(ctx, trunkBranch) + var others []string + if list, err := s.read(""); err != nil { + s.logf("the branches the source names are not read, and each is read at its next source-moved: %v", err) + } else { + others = list.Branches + } + for _, branch := range others { + if branch != trunkBranch { + s.take(ctx, branch) + } + } s.mu.Lock() s.caught = true branches := s.waiting @@ -76,17 +91,28 @@ func (s *sourceFollow) moved(ctx context.Context, branch string) { s.take(ctx, branch) } -// take reads one branch's source revision and applies it. A branch other than the trunk's is left until the indexer keeps -// a catalog graph per watched branch. +// take reads one branch's source revision (branch as the key name source. has it) and applies it: the trunk's +// through the take-in, another watched branch's into its own revision (A-93). func (s *sourceFollow) take(ctx context.Context, branch string) { - if branch != trunkBranch { - return - } a, err := s.read(branch) if err != nil { s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err) return } + if branch != trunkBranch { + name, err := keys.UnescapeName(branch) + if err != nil { + s.logf("source.%s names no branch: %v", branch, err) + return + } + if s.applyBranch == nil { + return + } + if err := s.applyBranch(ctx, name, a); err != nil { + s.logf("source.%s at revision %d could not be taken into the branch's revision: %v", branch, a.Revision, err) + } + return + } if err := s.apply(ctx, a); err != nil { s.logf("source.%s at revision %d could not be taken into the modules: %v", branch, a.Revision, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/branch_take_in.go b/modules/mesh-delivery-indexer/internal/index/branch_take_in.go new file mode 100644 index 00000000..82258df1 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/branch_take_in.go @@ -0,0 +1,55 @@ +package index + +import ( + "context" + "errors" + "maps" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// ApplyBranchSource takes a watched branch other than the first in from the source revision the seat answered for it +// (design 47 G §13, §14, §291, ADR 0666 §6, A-85, A-93): a pure computation from the merged messages of the heads the source +// names, over a view in memory seeded from the branch's newest revision, written straight into the branch's next revision +// (KeepBranchEntries): each module's fingerprint, inputs and graph node at the branch's heads, computed as the trunk's are +// (applyMoving). Nothing of the trunk is written: its modules, manifests, taken state, graph and packages stay its own, +// and the branch has no bucket of its own. A head whose merged message is not read keeps no revision, said; the next +// source-moved of the branch takes it in again. +func (ix *Indexer) ApplyBranchSource(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error { + if isTrunkBranch(branch) || ix.Revisions == nil || ix.Message == nil || len(a.Repositories) == 0 { + return nil + } + heads := map[string]string{} + for _, r := range a.Repositories { + heads[r.Repository] = r.Head + } + view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) + if errors.Is(err, process.ErrNotYet) { + ix.logf("catalog graph: the branch %s is not taken in yet: its objects %s are not on the bus", branch, CatalogRevisionsObjects) + return nil + } + if err != nil { + return err + } + d := ix.dry() + // The branch's own view: its modules and graph from its revision, no taken merge, and the packages it publishes in memory + // over the trunk's, never written to them. + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + if ix.Packages != nil { + d.ix.Packages = newOverlay(ix.Packages) + } + d.ix.source = sourceNow{rev: uint64(a.Revision), heads: maps.Clone(heads)} + d.ix.Logf = func(format string, args ...any) { + ix.logf("branch "+branch+", computed and not written: "+format, args...) + } + if unread := d.ix.takeInHeads(ctx, heads); len(unread) > 0 { + ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: %d repositories were not taken in (%v); taken in again at its next source-moved", branch, a.Revision, len(unread), unread) + return nil + } + entries, err := d.ix.viewEntries(ctx) + if err != nil { + return err + } + return ix.KeepBranchEntries(ctx, branch, a.Revision, entries) +} diff --git a/modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go b/modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go new file mode 100644 index 00000000..9c9ba9b8 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go @@ -0,0 +1,75 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "slices" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" +) + +// Design 47 G §13, §14, §291, A-93: a watched branch other than the first is taken in as a pure computation from the merged +// messages of its heads, straight into branch..; the trunk's modules, graph and revision stay the trunk's. +func TestABranchOtherThanTheFirstIsTakenInIntoItsOwnRevisionAndTheTrunkIsLeftAlone(t *testing.T) { + ctx := context.Background() + ix, mods, merged := newFormIndexerUnmerged(t, builderModules(t)...) + ix.Revisions = newMemObjects(0) + ix.Graph = newMem() + ix.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) { + if repository != catalogueRepo || head != merged.Merge { + return nil, fmt.Errorf("no merge of %s at %s", repository, head) + } + return json.Marshal(merged) + } + var said []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + said = append(said, e) + return nil + } + before := map[string]string{} + keys, _ := mods.Keys(ctx) + for _, k := range keys { + v, _, _, _ := mods.Get(ctx, k) + before[k] = string(v) + } + graphBefore, _ := ix.Graph.Keys(ctx) + answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}} + if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil { + t.Fatal(err) + } + if len(said) != 1 || said[0].Revision != 1 || said[0].Branch != "beta" || said[0].Source != 9 || said[0].Entries != int64(len(builderModules(t))) { + t.Fatalf("the branch take-in said %+v, want revision 1 of beta at source 9 with every module", said) + } + rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1)) + if err != nil || rev.Branch != "beta" || len(entries) != len(builderModules(t)) { + t.Fatalf("branch.beta.1 = %+v with %d entries (%v)", rev, len(entries), err) + } + for _, e := range entries { + if e.Commit != merged.Merge || e.Inputs == nil { // computed at the branch's head, with the inputs its fingerprint stands on + t.Errorf("the entry of %s is at %q with inputs %v, want the branch's head %s with inputs", e.Module, e.Commit, e.Inputs, merged.Merge) + } + } + if _, any, _ := newestTrunk(ctx, ix.Revisions); any { + t.Error("a branch take-in kept a trunk revision") + } + after := map[string]string{} + keys, _ = mods.Keys(ctx) + for _, k := range keys { + v, _, _, _ := mods.Get(ctx, k) + after[k] = string(v) + } + if !maps.Equal(before, after) { + t.Errorf("a branch take-in wrote the trunk's modules") + } + if graphAfter, _ := ix.Graph.Keys(ctx); !slices.Equal(graphBefore, graphAfter) { + t.Errorf("a branch take-in wrote the trunk's graph: %v, was %v", graphAfter, graphBefore) + } + // Taken in again at the same heads, nothing moved: no second revision. + if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil || len(said) != 1 { + t.Errorf("the same heads said %d revisions (%v), want one", len(said), err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index cea8ecec..cd999c4a 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -246,15 +246,25 @@ func (ix *Indexer) KeepView(ctx context.Context) error { if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() { return nil } - held, err := ix.heldModules(ctx) + entries, err := ix.viewEntries(ctx) if err != nil { return err } + return ix.keepEntries(ctx, entries) +} + +// viewEntries is the entry of every module the indexer's view (its modules and its module graph) holds: the module's value +// with its node of the graph, as KeepView keeps them. +func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.CatalogEntry, error) { + held, err := ix.heldModules(ctx) + if err != nil { + return nil, err + } nodes := graph.Graph{} if ix.Graph != nil { keys, err := ix.Graph.Keys(ctx) if err != nil && !errors.Is(err, process.ErrNotYet) { - return fmt.Errorf("the module graph: %w", err) + return nil, fmt.Errorf("the module graph: %w", err) } for _, k := range keys { if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found { @@ -275,7 +285,7 @@ func (ix *Indexer) KeepView(ctx context.Context) error { } entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, m, raw, nodes[name]), m) } - return ix.keepEntries(ctx, entries) + return entries, nil } // keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry @@ -411,8 +421,14 @@ func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) { // graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's // catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view. func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) { + return BranchView(ctx, o, TrunkBranch) +} + +// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch +// (design 47 G §291, A-93), empty and at 0 when it keeps none. +func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) { view, held := newOverlay(nil), newOverlay(nil) - newest, any, err := newestTrunk(ctx, o) + newest, any, err := newestOf(ctx, o, branch) if err != nil || !any { return view, held, 0, err }