DLV-I10: indexer_config_readers; and indexer_read_findings with its views event
indexer_config_readers {module} answers {readers: [{module, key, kind,
readers}]} for every value key and own secret any kept manifest of the
module ever declared: the module where its trunk's manifest declares it,
each open pull request whose head declares it, none for a key no longer
read (design 32 *A value no longer read*). It reads one watched copy of
the modules, manifests and config-candidates states. Nodes is left out:
the indexer cannot say which nodes' builds read a key. The indexer's
findings are said on its views family and read by indexer_read_findings,
for the condition keeper (the lead's ruling).
Issue 1966.
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -194,7 +194,7 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...)...)); err != nil {
|
||||
if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...), configReadersTool(stdio.NewWatchedStates()))...)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -30,3 +32,52 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now
|
||||
|
||||
// keptSource is one of the indexer's states as its runtime reads it.
|
||||
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
|
||||
|
||||
// configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy
|
||||
// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state
|
||||
// whole again.
|
||||
func configReadersTool(watched *stdio.WatchedStates) stdio.Tool {
|
||||
return stdio.Tool{Name: index.ConfigReadersTool,
|
||||
Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " +
|
||||
"the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " +
|
||||
"each open pull request (owner/repository#number) whose head declares it. A key with no reader is no longer read by any " +
|
||||
"trunk or pull request, which mesh-config retires; which nodes still run a build reading it is not the indexer's to say.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module whose config keys are asked"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
module, _ := args["module"].(string)
|
||||
if module == "" {
|
||||
return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool)
|
||||
}
|
||||
trunk := map[string]delivery.Module{}
|
||||
mods, err := watched.Values("modules")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the modules state cannot be read: %w", err)
|
||||
}
|
||||
for k, v := range mods {
|
||||
if m, err := delivery.DecodeModule(v); err == nil {
|
||||
trunk[k] = m
|
||||
}
|
||||
}
|
||||
kept := map[string]delivery.Manifest{}
|
||||
raws, err := watched.Values("manifests")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifests state cannot be read: %w", err)
|
||||
}
|
||||
for k, v := range raws {
|
||||
if m, err := delivery.DecodeManifest(v); err == nil {
|
||||
kept[k] = m
|
||||
}
|
||||
}
|
||||
var pulls []delivery.ConfigCandidate
|
||||
if cands, err := watched.Values(delivery.IndexerConfigCandidatesState); err == nil {
|
||||
for _, v := range cands {
|
||||
var c delivery.ConfigCandidate
|
||||
if json.Unmarshal(v, &c) == nil {
|
||||
pulls = append(pulls, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
return map[string]any{"readers": index.ConfigReaders(module, trunk, kept, pulls)}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -500,7 +500,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates),
|
||||
index.VerdictsState: awaited(verdicts), delivery.IndexerPredefinedState: awaited(predefined),
|
||||
delivery.IndexerConfigCandidatesState: awaited(configCandidates), delivery.PackagesState: awaited(packages),
|
||||
graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken)}, errs, logger.Printf)
|
||||
graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings)}, errs, logger.Printf)
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger); ix.KickFill(ctx) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ConfigReadersTool is the indexer's tool that says who reads a module's config keys (design 32 *A value no longer
|
||||
// read*): mesh-config asks it on each manifest-accepted, and a key no manifest on any trunk or open pull request reads
|
||||
// any more is a retiring value.
|
||||
const ConfigReadersTool = "indexer_config_readers"
|
||||
|
||||
// The kinds of a config key.
|
||||
const (
|
||||
ConfigKindValue = "value"
|
||||
ConfigKindSecret = "secret"
|
||||
)
|
||||
|
||||
// ConfigReaderRow is one key of a module's config: its kind (a value, or one of its own secrets) and who reads it now —
|
||||
// the module itself where its trunk's manifest declares it, and each open pull request (owner/repository#number) whose
|
||||
// head declares it. Nodes is left out: which nodes' applied builds read a key is the node-engines' word, not the
|
||||
// indexer's, so mesh-config reads it there (an empty list would say "no node", which the indexer cannot know).
|
||||
type ConfigReaderRow struct {
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
Readers []string `json:"readers"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
}
|
||||
|
||||
// configKeys are a manifest's config value keys (its `config` or `settings`) and its own secrets' names.
|
||||
func configKeys(raw string) (values, secrets []string) {
|
||||
if d, err := DeclarationsOf(raw); err == nil {
|
||||
for k := range d {
|
||||
values = append(values, k)
|
||||
}
|
||||
}
|
||||
var m struct {
|
||||
OwnSecrets map[string]json.RawMessage `json:"own-secrets"`
|
||||
}
|
||||
if json.Unmarshal([]byte(raw), &m) == nil {
|
||||
for k := range m.OwnSecrets {
|
||||
secrets = append(secrets, k)
|
||||
}
|
||||
}
|
||||
return values, secrets
|
||||
}
|
||||
|
||||
// ConfigReaders are the rows of module's config keys: every key any manifest of the module the indexer keeps ever
|
||||
// declared (so a key no manifest declares any more is a row with no reader), each with who reads it now. trunk is the
|
||||
// modules state, manifests the manifests state by digest, candidates the open pull requests' config declarations.
|
||||
func ConfigReaders(module string, trunk map[string]delivery.Module, manifests map[string]delivery.Manifest, candidates []delivery.ConfigCandidate) []ConfigReaderRow {
|
||||
rows := map[string]*ConfigReaderRow{}
|
||||
add := func(kind, key, reader string) {
|
||||
id := kind + "|" + key
|
||||
r := rows[id]
|
||||
if r == nil {
|
||||
r = &ConfigReaderRow{Module: module, Key: key, Kind: kind, Readers: []string{}}
|
||||
rows[id] = r
|
||||
}
|
||||
if reader != "" && !slices.Contains(r.Readers, reader) {
|
||||
r.Readers = append(r.Readers, reader)
|
||||
}
|
||||
}
|
||||
current := ""
|
||||
if m, ok := trunk[module]; ok {
|
||||
current = m.Manifest
|
||||
}
|
||||
for digest, kept := range manifests {
|
||||
if kept.Module != module {
|
||||
continue
|
||||
}
|
||||
reader := ""
|
||||
if digest == current {
|
||||
reader = module
|
||||
}
|
||||
values, secrets := configKeys(kept.JSON)
|
||||
for _, k := range values {
|
||||
add(ConfigKindValue, k, reader)
|
||||
}
|
||||
for _, k := range secrets {
|
||||
add(ConfigKindSecret, k, reader)
|
||||
}
|
||||
}
|
||||
for _, c := range candidates {
|
||||
for key := range c.Declares[module] {
|
||||
add(ConfigKindValue, key, c.Pull)
|
||||
}
|
||||
}
|
||||
out := make([]ConfigReaderRow, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
slices.Sort(r.Readers)
|
||||
out = append(out, *r)
|
||||
}
|
||||
slices.SortFunc(out, func(a, b ConfigReaderRow) int {
|
||||
if a.Kind != b.Kind {
|
||||
if a.Kind < b.Kind {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
if a.Key < b.Key {
|
||||
return -1
|
||||
}
|
||||
if a.Key > b.Key {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package index
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -15,6 +16,27 @@ import (
|
||||
// than the one kept never replacing it; and each open pull request's config declarations its head adds or changes against the
|
||||
// trunk's manifest, an empty set when none, written only when they change and deleted when it merges or closes.
|
||||
func TestTheControlRecords(t *testing.T) {
|
||||
// DLV-I10 (design 32 *A value no longer read*): every key any kept manifest of the module declared, each with who
|
||||
// reads it now: the module for its trunk's manifest, an open pull request for its head's declarations; a key no
|
||||
// manifest declares now has no reader.
|
||||
t.Run("indexer_config_readers", func(t *testing.T) {
|
||||
d1, d2 := strings.Repeat("1", 64), strings.Repeat("2", 64)
|
||||
manifests := map[string]delivery.Manifest{
|
||||
d1: {Module: "web", JSON: `{"module":"web","config":{"old":{"kind":"value","why":"w"},"port":{"kind":"value","why":"w"}},"own-secrets":{"key":"x"}}`},
|
||||
d2: {Module: "web", JSON: `{"module":"web","config":{"port":{"kind":"value","why":"w"}}}`},
|
||||
strings.Repeat("3", 64): {Module: "db", JSON: `{"module":"db","config":{"old":{"kind":"value","why":"w"}}}`},
|
||||
}
|
||||
trunk := map[string]delivery.Module{"web": {Manifest: d2}}
|
||||
pulls := []delivery.ConfigCandidate{{Pull: "novox/web#7", Declares: map[string]map[string]delivery.ConfigDeclaration{"web": {"new": {}}}}}
|
||||
got := ConfigReaders("web", trunk, manifests, pulls)
|
||||
want := []ConfigReaderRow{{Module: "web", Key: "key", Kind: ConfigKindSecret, Readers: []string{}},
|
||||
{Module: "web", Key: "new", Kind: ConfigKindValue, Readers: []string{"novox/web#7"}},
|
||||
{Module: "web", Key: "old", Kind: ConfigKindValue, Readers: []string{}},
|
||||
{Module: "web", Key: "port", Kind: ConfigKindValue, Readers: []string{"web"}}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("rows = %+v\nwant %+v", got, want)
|
||||
}
|
||||
})
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
||||
t.Run("the predefined list, never replaced by an older merge", func(t *testing.T) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package index
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/views"
|
||||
@@ -33,6 +34,7 @@ var ReadTools = []ViewedTool{
|
||||
{Tool: "indexer_read_graph", State: graph.State, Words: "the module graph: each module's build and provision edges."},
|
||||
{Tool: "indexer_read_heard", State: HeardState, Words: "the forge messages the indexer heard and what it did with each."},
|
||||
{Tool: "indexer_read_taken", State: "taken", Words: "each repository's merge the indexer took in last."},
|
||||
{Tool: "indexer_read_findings", State: health.FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."},
|
||||
}
|
||||
|
||||
// ViewedTool is one tool over one state of the indexer's.
|
||||
@@ -52,4 +54,5 @@ var ViewedStates = map[string]string{
|
||||
graph.State: graph.Bucket,
|
||||
HeardState: HeardBucket,
|
||||
"taken": delivery.TakenBucket,
|
||||
health.FindingsState: health.FindingsBucket(delivery.IndexerModule),
|
||||
}
|
||||
|
||||
@@ -149,8 +149,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State)
|
||||
}
|
||||
}
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again") // DLV-F3r
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 18 {
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool) // DLV-F3r, DLV-I10
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 20 {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
|
||||
|
||||
@@ -32,8 +32,10 @@
|
||||
"indexer_read_graph",
|
||||
"indexer_read_heard",
|
||||
"indexer_read_taken",
|
||||
"indexer_read_findings",
|
||||
"indexer_first_views",
|
||||
"indexer_first_view_again"
|
||||
"indexer_first_view_again",
|
||||
"indexer_config_readers"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
|
||||
Reference in New Issue
Block a user