diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 6eb57494..36e77d5a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -194,7 +194,7 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) { } func main() { - if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...)...)); err != nil { + if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...), configReadersTool(stdio.NewWatchedStates()))...)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go index 33717966..c86c82e0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go @@ -1,6 +1,8 @@ package main import ( + "encoding/json" + "fmt" "time" stdio "git.novox.be/novox/mesh-sdk/go" @@ -30,3 +32,52 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now // keptSource is one of the indexer's states as its runtime reads it. func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} } + +// configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy +// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state +// whole again. +func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { + return stdio.Tool{Name: index.ConfigReadersTool, + Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " + + "the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " + + "each open pull request (owner/repository#number) whose head declares it. A key with no reader is no longer read by any " + + "trunk or pull request, which mesh-config retires; which nodes still run a build reading it is not the indexer's to say.", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module whose config keys are asked"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + if module == "" { + return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool) + } + trunk := map[string]delivery.Module{} + mods, err := watched.Values("modules") + if err != nil { + return nil, fmt.Errorf("the modules state cannot be read: %w", err) + } + for k, v := range mods { + if m, err := delivery.DecodeModule(v); err == nil { + trunk[k] = m + } + } + kept := map[string]delivery.Manifest{} + raws, err := watched.Values("manifests") + if err != nil { + return nil, fmt.Errorf("the manifests state cannot be read: %w", err) + } + for k, v := range raws { + if m, err := delivery.DecodeManifest(v); err == nil { + kept[k] = m + } + } + var pulls []delivery.ConfigCandidate + if cands, err := watched.Values(delivery.IndexerConfigCandidatesState); err == nil { + for _, v := range cands { + var c delivery.ConfigCandidate + if json.Unmarshal(v, &c) == nil { + pulls = append(pulls, c) + } + } + } + return map[string]any{"readers": index.ConfigReaders(module, trunk, kept, pulls)}, nil + }} +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 4e64eb90..c8b7f745 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -500,7 +500,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates), index.VerdictsState: awaited(verdicts), delivery.IndexerPredefinedState: awaited(predefined), delivery.IndexerConfigCandidatesState: awaited(configCandidates), delivery.PackagesState: awaited(packages), - graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken)}, errs, logger.Printf) + graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings)}, errs, logger.Printf) cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger); ix.KickFill(ctx) }, logger.Printf) if err != nil { return err diff --git a/modules/mesh-delivery-indexer/internal/index/config_readers.go b/modules/mesh-delivery-indexer/internal/index/config_readers.go new file mode 100644 index 00000000..df4e4311 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/config_readers.go @@ -0,0 +1,113 @@ +package index + +import ( + "encoding/json" + "slices" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ConfigReadersTool is the indexer's tool that says who reads a module's config keys (design 32 *A value no longer +// read*): mesh-config asks it on each manifest-accepted, and a key no manifest on any trunk or open pull request reads +// any more is a retiring value. +const ConfigReadersTool = "indexer_config_readers" + +// The kinds of a config key. +const ( + ConfigKindValue = "value" + ConfigKindSecret = "secret" +) + +// ConfigReaderRow is one key of a module's config: its kind (a value, or one of its own secrets) and who reads it now — +// the module itself where its trunk's manifest declares it, and each open pull request (owner/repository#number) whose +// head declares it. Nodes is left out: which nodes' applied builds read a key is the node-engines' word, not the +// indexer's, so mesh-config reads it there (an empty list would say "no node", which the indexer cannot know). +type ConfigReaderRow struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Readers []string `json:"readers"` + Nodes []string `json:"nodes,omitempty"` +} + +// configKeys are a manifest's config value keys (its `config` or `settings`) and its own secrets' names. +func configKeys(raw string) (values, secrets []string) { + if d, err := DeclarationsOf(raw); err == nil { + for k := range d { + values = append(values, k) + } + } + var m struct { + OwnSecrets map[string]json.RawMessage `json:"own-secrets"` + } + if json.Unmarshal([]byte(raw), &m) == nil { + for k := range m.OwnSecrets { + secrets = append(secrets, k) + } + } + return values, secrets +} + +// ConfigReaders are the rows of module's config keys: every key any manifest of the module the indexer keeps ever +// declared (so a key no manifest declares any more is a row with no reader), each with who reads it now. trunk is the +// modules state, manifests the manifests state by digest, candidates the open pull requests' config declarations. +func ConfigReaders(module string, trunk map[string]delivery.Module, manifests map[string]delivery.Manifest, candidates []delivery.ConfigCandidate) []ConfigReaderRow { + rows := map[string]*ConfigReaderRow{} + add := func(kind, key, reader string) { + id := kind + "|" + key + r := rows[id] + if r == nil { + r = &ConfigReaderRow{Module: module, Key: key, Kind: kind, Readers: []string{}} + rows[id] = r + } + if reader != "" && !slices.Contains(r.Readers, reader) { + r.Readers = append(r.Readers, reader) + } + } + current := "" + if m, ok := trunk[module]; ok { + current = m.Manifest + } + for digest, kept := range manifests { + if kept.Module != module { + continue + } + reader := "" + if digest == current { + reader = module + } + values, secrets := configKeys(kept.JSON) + for _, k := range values { + add(ConfigKindValue, k, reader) + } + for _, k := range secrets { + add(ConfigKindSecret, k, reader) + } + } + for _, c := range candidates { + for key := range c.Declares[module] { + add(ConfigKindValue, key, c.Pull) + } + } + out := make([]ConfigReaderRow, 0, len(rows)) + for _, r := range rows { + slices.Sort(r.Readers) + out = append(out, *r) + } + slices.SortFunc(out, func(a, b ConfigReaderRow) int { + if a.Kind != b.Kind { + if a.Kind < b.Kind { + return -1 + } + return 1 + } + if a.Key < b.Key { + return -1 + } + if a.Key > b.Key { + return 1 + } + return 0 + }) + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 119a7fe8..b8b8a0da 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -3,6 +3,7 @@ package index import ( "context" "encoding/json" + "reflect" "strings" "testing" "time" @@ -15,6 +16,27 @@ import ( // than the one kept never replacing it; and each open pull request's config declarations its head adds or changes against the // trunk's manifest, an empty set when none, written only when they change and deleted when it merges or closes. func TestTheControlRecords(t *testing.T) { + // DLV-I10 (design 32 *A value no longer read*): every key any kept manifest of the module declared, each with who + // reads it now: the module for its trunk's manifest, an open pull request for its head's declarations; a key no + // manifest declares now has no reader. + t.Run("indexer_config_readers", func(t *testing.T) { + d1, d2 := strings.Repeat("1", 64), strings.Repeat("2", 64) + manifests := map[string]delivery.Manifest{ + d1: {Module: "web", JSON: `{"module":"web","config":{"old":{"kind":"value","why":"w"},"port":{"kind":"value","why":"w"}},"own-secrets":{"key":"x"}}`}, + d2: {Module: "web", JSON: `{"module":"web","config":{"port":{"kind":"value","why":"w"}}}`}, + strings.Repeat("3", 64): {Module: "db", JSON: `{"module":"db","config":{"old":{"kind":"value","why":"w"}}}`}, + } + trunk := map[string]delivery.Module{"web": {Manifest: d2}} + pulls := []delivery.ConfigCandidate{{Pull: "novox/web#7", Declares: map[string]map[string]delivery.ConfigDeclaration{"web": {"new": {}}}}} + got := ConfigReaders("web", trunk, manifests, pulls) + want := []ConfigReaderRow{{Module: "web", Key: "key", Kind: ConfigKindSecret, Readers: []string{}}, + {Module: "web", Key: "new", Kind: ConfigKindValue, Readers: []string{"novox/web#7"}}, + {Module: "web", Key: "old", Kind: ConfigKindValue, Readers: []string{}}, + {Module: "web", Key: "port", Kind: ConfigKindValue, Readers: []string{"web"}}} + if !reflect.DeepEqual(got, want) { + t.Errorf("rows = %+v\nwant %+v", got, want) + } + }) ctx := context.Background() at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) t.Run("the predefined list, never replaced by an older merge", func(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index a66f9ef7..ee2b7f9c 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -2,6 +2,7 @@ package index import ( "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-delivery/sdk/views" @@ -33,6 +34,7 @@ var ReadTools = []ViewedTool{ {Tool: "indexer_read_graph", State: graph.State, Words: "the module graph: each module's build and provision edges."}, {Tool: "indexer_read_heard", State: HeardState, Words: "the forge messages the indexer heard and what it did with each."}, {Tool: "indexer_read_taken", State: "taken", Words: "each repository's merge the indexer took in last."}, + {Tool: "indexer_read_findings", State: health.FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."}, } // ViewedTool is one tool over one state of the indexer's. @@ -52,4 +54,5 @@ var ViewedStates = map[string]string{ graph.State: graph.Bucket, HeardState: HeardBucket, "taken": delivery.TakenBucket, + health.FindingsState: health.FindingsBucket(delivery.IndexerModule), } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 5d82fa35..478af047 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -149,8 +149,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State) } } - want = append(want, "indexer_first_views", "indexer_first_view_again") // DLV-F3r - if !reflect.DeepEqual(m.Tools, want) || len(want) != 18 { + want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool) // DLV-F3r, DLV-I10 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 20 { t.Errorf("tools = %v, want %v", m.Tools, want) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 2b42a1e9..2d314f28 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -32,8 +32,10 @@ "indexer_read_graph", "indexer_read_heard", "indexer_read_taken", + "indexer_read_findings", "indexer_first_views", - "indexer_first_view_again" + "indexer_first_view_again", + "indexer_config_readers" ], "consumes": [ "mesh-build-go.toolchain.reported",