diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 98bdb4a1..9446f278 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -194,7 +194,7 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) { } func main() { - if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...), configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource))...)); err != nil { + if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...), configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource))...)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go index e65c0dfd..88612120 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go @@ -94,3 +94,34 @@ func busStateTool(state func(string) views.Source) stdio.Tool { return index.BusStateOf(state("modules"), state(index.VerdictsState), state("manifests"), state(views.MarksState)) }} } + +// requirementsTool is indexer_requirements (REG S15; DLV-I7): the versions of one mesh Go module its consumers' go.sum +// files require, from the indexer's requirements state, kept at each merge. The builder's floor numbers no version below +// the highest. +func requirementsTool(state func(string) views.Source) stdio.Tool { + return stdio.Tool{Name: index.RequirementsTool, + Description: "The versions of one mesh Go module (its module path, as go.sum names it) that the trunk's consumers require: " + + "the highest each consumer's go.sum names, distinct, highest last, kept by the indexer at each merge. The builder " + + "numbers no version of the package below the highest.", + Input: map[string]any{"type": "object", "required": []string{"package"}, "properties": map[string]any{ + "package": map[string]any{"type": "string", "description": "the mesh Go module's path, such as git.novox.be/novox/mesh-sdk/go"}}}, + Run: func(args map[string]any) (any, error) { + pkg, _ := args["package"].(string) + if pkg == "" { + return nil, fmt.Errorf("%s names the package", index.RequirementsTool) + } + e, found, err := state(index.RequirementsState).Get(pkg) + if err != nil { + return nil, fmt.Errorf("the requirements state cannot be read: %w", err) + } + var value []byte + if found { + value = e.Value + } + versions, err := index.RequirementsOf(value, found) + if err != nil { + return nil, err + } + return map[string]any{"package": pkg, "requirements": versions}, nil + }} +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index c8b7f745..767449fe 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -325,9 +325,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then // the judge refuses no use of an old section. oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf) + // What each consumer's go.sum requires of each mesh Go module (DLV-I7): a state this delivery brings, bound when it + // appears; until then indexer_requirements answers none, and the builder's floor waits on its answer. + requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf) // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf) - ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Requirements: awaitedStore{requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 85d1c415..9f8562b7 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -57,6 +57,11 @@ type Indexer struct { // OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no // count and the judge refuses no use of an old section. OldSections Store + // Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh + // Go module, kept at each merge; nil keeps none. + Requirements Store + reqMu sync.Mutex + reqKept map[string]bool // consumers whose go.sum this process kept (requirementsKept) // JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's. JudgeID string // SayRefused says the event manifest-refused for a refusal newly written; nil says nothing. diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index ecb709a8..55d50126 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -82,6 +82,33 @@ func TestTheControlRecords(t *testing.T) { t.Errorf("indexer_bus_state = %+v (%v)", got, err) } }) + // DLV-I7 (REG S15, the lead's ruling): the requirements state keeps, per mesh Go module, the highest version each + // consumer's go.sum names; a consumer no longer naming it leaves; indexer_requirements answers the distinct versions, + // highest last, a pseudo-version below the release it precedes. + t.Run("indexer_requirements", func(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Requirements: newMem()} + sdk := "git.novox.be/novox/mesh-sdk/go" + sum := func(lines ...string) []byte { return []byte(strings.Join(lines, "\n")) } + ix.keepRequirements(ctx, "novox/a", "modules/x", sum( + sdk+" v0.1.14-0.20261103221822-85ba0a9a70d8 h1:x=", sdk+" v0.1.13/go.mod h1:y=", "github.com/other v1.0.0 h1:z=")) + ix.keepRequirements(ctx, "novox/b", ".", sum(sdk+" v0.1.14 h1:x=")) + ix.keepRequirements(ctx, "novox/c", ".", sum(sdk+" v0.1.14 h1:x=")) + value, _, found, _ := ix.Requirements.Get(ctx, sdk) + got, err := RequirementsOf(value, found) + if err != nil || !reflect.DeepEqual(got, []string{"v0.1.14-0.20261103221822-85ba0a9a70d8", "v0.1.14"}) { + t.Errorf("requirements = %v (%v)", got, err) + } + ix.keepRequirements(ctx, "novox/a", "modules/x", sum("github.com/other v1.0.0 h1:z=")) + ix.keepRequirements(ctx, "novox/b", ".", nil) + ix.keepRequirements(ctx, "novox/c", ".", nil) + if _, _, found, _ := ix.Requirements.Get(ctx, sdk); found { + t.Error("a package no consumer requires is still kept") + } + if got, _ := RequirementsOf(nil, false); got == nil || len(got) != 0 { + t.Errorf("none = %#v, want an empty list", got) + } + }) ctx := context.Background() at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) t.Run("the predefined list, never replaced by an older merge", func(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 241bd88e..6612ff0f 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -403,7 +403,7 @@ func (ix *Indexer) otherPackages(ctx context.Context, f delivery.Forge, dir stri if hash == "" { return nil } - if has && cur.Inputs2 != nil && delivery.SameRepository(cur.Repository, f.Repository) { + if has && cur.Inputs2 != nil && delivery.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) { for _, h := range cur.Inputs2.Locks { if h == hash { return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository) @@ -424,5 +424,6 @@ func (ix *Indexer) otherPackages(ctx context.Context, f delivery.Forge, dir stri } return nil } + ix.keepRequirements(ctx, f.Repository, dir, raw) return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository) } diff --git a/modules/mesh-delivery-indexer/internal/index/requirements.go b/modules/mesh-delivery-indexer/internal/index/requirements.go new file mode 100644 index 00000000..6f9e530f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/requirements.go @@ -0,0 +1,228 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "slices" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The requirements state (DLV-I7, the lead's ruling on I7; REG S15, ADR 0606 §1): per mesh Go module, the highest +// version of it each consumer's go.sum names, filled at each merge from the directories whose go.sum the indexer reads. +// The builder's floor asks it through indexer_requirements, so a version is never numbered below what a consumer +// already requires. +const ( + RequirementsState = "requirements" + RequirementsBucket = delivery.IndexerModule + "_" + RequirementsState + RequirementsTool = "indexer_requirements" +) + +// Requirement is the value of one package's key in the requirements state: By maps each consumer, ":", +// to the highest version of Package its go.sum names. +type Requirement struct { + V int `json:"v"` + Package string `json:"package"` + By map[string]string `json:"by"` +} + +// Versions are the distinct versions consumers require, highest last (CompareVersions). +func (r Requirement) Versions() []string { + seen := map[string]bool{} + for _, v := range r.By { + seen[v] = true + } + out := slices.Collect(maps.Keys(seen)) + slices.SortFunc(out, CompareVersions) + return out +} + +// RequiredOf is, per mesh Go module a go.sum names (as fingerprint.MeshPackagesOf takes them), the highest version it +// names, its module line or its go.mod line alike. +func RequiredOf(goSum []byte) map[string]string { + out := map[string]string{} + for _, line := range strings.Split(string(goSum), "\n") { + f := strings.Fields(line) + if len(f) < 2 || !strings.HasPrefix(f[0], delivery.ForgeModulePrefix) { + continue + } + if _, _, ok := delivery.PackageRepository(f[0]); !ok { + continue + } + v := strings.TrimSuffix(f[1], "/go.mod") + if cur, ok := out[f[0]]; !ok || CompareVersions(cur, v) < 0 { + out[f[0]] = v + } + } + return out +} + +// CompareVersions orders two Go module versions as semantic versions (vMAJOR.MINOR.PATCH[-pre][+build]); a +// pseudo-version is a pre-release, so it sorts below the release it precedes. One that does not read sorts first. +func CompareVersions(a, b string) int { + pa, oka := semverParts(a) + pb, okb := semverParts(b) + switch { + case !oka || !okb: + if oka != okb { + if oka { + return 1 + } + return -1 + } + return strings.Compare(a, b) + } + for i := 0; i < 3; i++ { + if c := cmpNum(pa.core[i], pb.core[i]); c != 0 { + return c + } + } + switch { + case pa.pre == "" && pb.pre == "": + return 0 + case pa.pre == "": + return 1 + case pb.pre == "": + return -1 + } + xa, xb := strings.Split(pa.pre, "."), strings.Split(pb.pre, ".") + for i := 0; i < len(xa) && i < len(xb); i++ { + na, ea := strconv.ParseUint(xa[i], 10, 64) + nb, eb := strconv.ParseUint(xb[i], 10, 64) + switch { + case ea == nil && eb == nil: + if na != nb { + return cmpNum(na, nb) + } + case ea == nil: + return -1 + case eb == nil: + return 1 + default: + if c := strings.Compare(xa[i], xb[i]); c != 0 { + return c + } + } + } + return cmpNum(uint64(len(xa)), uint64(len(xb))) +} + +type semver struct { + core [3]uint64 + pre string +} + +func semverParts(v string) (semver, bool) { + var s semver + if !strings.HasPrefix(v, "v") { + return s, false + } + v, _, _ = strings.Cut(v[1:], "+") + core, pre, _ := strings.Cut(v, "-") + parts := strings.Split(core, ".") + if len(parts) != 3 { + return s, false + } + for i, p := range parts { + n, err := strconv.ParseUint(p, 10, 64) + if err != nil { + return s, false + } + s.core[i] = n + } + s.pre = pre + return s, true +} + +func cmpNum(a, b uint64) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + } + return 0 +} + +// keepRequirements writes what the go.sum of repository's directory dir requires into the requirements state: each +// package it names gets the consumer's version, and a package it no longer names loses the consumer. nil +// Requirements keeps nothing. A write that fails is said in the log; the next merge of the directory writes it again. +func (ix *Indexer) keepRequirements(ctx context.Context, repository, dir string, goSum []byte) { + if ix.Requirements == nil { + return + } + consumer := repository + ":" + dir + ix.reqMu.Lock() + if ix.reqKept == nil { + ix.reqKept = map[string]bool{} + } + ix.reqKept[consumer] = true + ix.reqMu.Unlock() + required := RequiredOf(goSum) + keys, err := ix.Requirements.Keys(ctx) + if err != nil { + ix.logf("requirements: the state cannot be listed, so what %s requires is not kept: %v", consumer, err) + return + } + for _, pkg := range slices.Sorted(maps.Keys(required)) { + if !slices.Contains(keys, pkg) { + keys = append(keys, pkg) + } + } + for _, pkg := range keys { + version, names := required[pkg] + err := write(ctx, ix.Requirements, pkg, func(value []byte, found bool) ([]byte, bool, error) { + r := Requirement{V: delivery.Version, Package: pkg, By: map[string]string{}} + if found && json.Unmarshal(value, &r) != nil { + r = Requirement{V: delivery.Version, Package: pkg, By: map[string]string{}} + } + if r.By == nil { + r.By = map[string]string{} + } + if cur, has := r.By[consumer]; names && cur == version || !names && !has { + return nil, false, nil + } + if names { + r.By[consumer] = version + } else { + delete(r.By, consumer) + } + if len(r.By) == 0 { + return nil, true, nil + } + next, err := json.Marshal(r) + return next, false, err + }) + if err != nil { + ix.logf("requirements: %s for %s is not kept: %v", pkg, consumer, err) + } + } +} + +// RequirementsOf answers indexer_requirements for one package from the state's value: the distinct versions its +// consumers require, highest last; none is an empty list. +func RequirementsOf(value []byte, found bool) ([]string, error) { + if !found { + return []string{}, nil + } + var r Requirement + if err := json.Unmarshal(value, &r); err != nil { + return nil, fmt.Errorf("the requirements value does not read: %w", err) + } + return r.Versions(), nil +} + +// requirementsKept says this process has kept what the directory's go.sum requires: until it has, the go.sum is read +// even where its hash is unchanged, so a consumer not merged since the state appeared is still kept at its next message. +func (ix *Indexer) requirementsKept(repository, dir string) bool { + if ix.Requirements == nil { + return true + } + ix.reqMu.Lock() + defer ix.reqMu.Unlock() + return ix.reqKept[repository+":"+dir] +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 1ae67786..71cb3ef8 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -113,7 +113,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, - {Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: health.FindingsState, History: 5}, {Name: views.StatusState, History: 1}}; !reflect.DeepEqual(m.State, want) { + {Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: health.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, + {Name: index.RequirementsState, History: 5}}; !reflect.DeepEqual(m.State, want) { // DLV-I7 t.Errorf("state = %+v, want %+v", m.State, want) } // DLV-F1r (ADR 0615 §1): the assigner's placement is read through its first view and views family, never its state. @@ -149,8 +150,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State) } } - want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool) // DLV-F3r, DLV-I10, DLV-I11 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 21 { + want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool) // DLV-F3r, DLV-I10, DLV-I11, DLV-I7 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 22 { t.Errorf("tools = %v, want %v", m.Tools, want) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 926c6704..e19baaf6 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -36,7 +36,8 @@ "indexer_first_views", "indexer_first_view_again", "indexer_config_readers", - "indexer_bus_state" + "indexer_bus_state", + "indexer_requirements" ], "consumes": [ "mesh-build-go.toolchain.reported", @@ -60,7 +61,8 @@ { "name": "graph", "history": 5 }, { "name": "view-marks", "history": 1 }, { "name": "findings", "history": 5 }, - { "name": "first-views", "history": 1 } + { "name": "first-views", "history": 1 }, + { "name": "requirements", "history": 5 } ], "streams": [ { "name": "indexer-views", "events": ["views.**"], "max-age-days": 7, "max-mib": 256 }